diff --git a/.acceptance.goreleaser.yml b/.acceptance.goreleaser.yml index f96f03d5..b3e10084 100644 --- a/.acceptance.goreleaser.yml +++ b/.acceptance.goreleaser.yml @@ -12,10 +12,15 @@ builds: env: - CC=x86_64-linux-gnu-gcc - CXX=x86_64-linux-gnu-g++ + - CGO_ENABLED=1 + - GOFLAGS= + - CGO_LDFLAGS= ldflags: - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser + tags: [] + archives: - id: homebrew format: tar.gz diff --git a/.cursor b/.cursor.bak similarity index 100% rename from .cursor rename to .cursor.bak diff --git a/.cursor/rules/acceptance-tests.mdc b/.cursor/rules/acceptance-tests.mdc new file mode 100644 index 00000000..d75337d6 --- /dev/null +++ b/.cursor/rules/acceptance-tests.mdc @@ -0,0 +1,44 @@ +--- +description: +globs: tests/acceptance/** +alwaysApply: false +--- +# Acceptance Tests Structure and Best Practices + +Acceptance tests are located in the [tests/acceptance/test_files/](mdc:tailpipe/tests/acceptance/test_files) directory. Each test case should be isolated in its own file for clarity and maintainability. + +## Guidelines for Acceptance Tests + +- **Test Isolation:** Each test should be in a separate file and use unique resource/config names to avoid conflicts. +- **Setup and Teardown:** Use `setup` and `teardown` functions to clean up config and data before and after each test, ensuring a clean environment. +- **Error Handling:** Focus assertions on user-facing error messages and observable outcomes, not on internal or debug messages that may change. +- **Clarity:** Use descriptive test names and comments to explain the purpose and expected outcome of each test. +- **Execution:** Tests should be runnable individually or in batches using the `run.sh` script from the `test_files` directory. +- **Framework:** All tests use [BATS](mdc:tailpipe/tailpipe/tailpipe/tailpipe/tailpipe/tailpipe/tailpipe/https:/github.com/bats-core/bats-core) for shell-based testing. +- **CI/CD Integration:** After adding new test files, update the test matrix in [.github/workflows/11-test-acceptance.yaml](mdc:tailpipe/.github/workflows/11-test-acceptance.yaml) to include the new test file in the acceptance test suite. +- **Command Dependencies:** When adding new tests that require additional shell commands or tools, update the `required_commands` array in `run.sh` to include the new dependencies. This ensures proper dependency checking in CI environments. +- **Test Verification:** Always run the test using `./tests/acceptance/run.sh filename.bats` after adding or modifying a test file to verify it works as expected. + +## Running Tests + +Tests must be run from the tailpipe root directory using the following command format: +```bash +./tests/acceptance/run.sh filename.bats +``` + +For example: +```bash +./tests/acceptance/run.sh partition_delete.bats +``` + +## Example Test Files + +- [partition_row_count.bats](mdc:tailpipe/tests/acceptance/test_files/partition_row_count.bats): Example of verifying a specific row count. +- [partition_filter.bats](mdc:tailpipe/tests/acceptance/test_files/partition_filter.bats): Example of verifying filter functionality. +- [partition_invalid_filter.bats](mdc:tailpipe/tests/acceptance/test_files/partition_invalid_filter.bats): Example of checking for clear error messages on invalid input. +- [partition_duplicate.bats](mdc:tailpipe/tests/acceptance/test_files/partition_duplicate.bats): Example of handling duplicate resource errors. +- [partition_invalid_source.bats](mdc:tailpipe/tests/acceptance/test_files/partition_invalid_source.bats): Example of handling references to non-existent resources. +- [partition_delete.bats](mdc:tailpipe/tests/acceptance/test_files/partition_delete.bats): Example of testing partition deletion and error handling for non-existent partitions. + +Use these examples as templates for new acceptance tests, adapting the structure and best practices for other features and scenarios. + diff --git a/.cursor/rules/general.mdc b/.cursor/rules/general.mdc new file mode 100644 index 00000000..1e956cb2 --- /dev/null +++ b/.cursor/rules/general.mdc @@ -0,0 +1,20 @@ +--- +description: +globs: +alwaysApply: false +--- +# general rules to always apply +## confirmation/avoid too much initiative +- DO not make any change I have not explicitly asked for +- NEVER make any changes if I have only asked you a question but not explicitly asked you to make an action +- Ask for confirmation before making ANY changes, with a summary of what you will do +## format +- Use lower case for sql always +## general attitude +- Use a neutral tone of voice and do not be too positive/enthusiastic. + - When I report a problem, do NOT say "perfect I see the problem" as that sounds like you know the solution + - When you have made a change do NOT say "now everything will be working" until you have confirmation that it does work + - Always look at my ideas and suggestions critically and look for flaws in my logic + - + + \ No newline at end of file diff --git a/.cursor/rules/project-context.mdc b/.cursor/rules/project-context.mdc new file mode 100644 index 00000000..a2c32e06 --- /dev/null +++ b/.cursor/rules/project-context.mdc @@ -0,0 +1,102 @@ +--- +description: +globs: +alwaysApply: true +--- +# Project Context and Guidelines + +## Project Overview +This is the main Tailpipe application that orchestrates data collection, transformation, and processing. It manages the plugin ecosystem, handles data flow between components, and provides the core infrastructure for data collection and processing pipelines. For more information about Tailpipe's capabilities and usage, see the [official documentation](mdc:tailpipe/https:/tailpipe.io/docs). + +## Related Repositories and Dependencies +- tailpipe-plugin-core: Core plugin functionality +- tailpipe-plugin-sdk: SDK for building plugins +- pipe-fittings: Data transformation components +- DuckDB: Used for SQL operations and data transformations +- Go standard library: Core functionality +- Other dependencies as specified in go.mod + +## Test Table Configuration +When writing tests that require custom tables, follow these guidelines. For detailed information about custom tables, see the [custom tables documentation](mdc:tailpipe/https:/github.com/turbot/tailpipe-plugin-sdk/blob/6ad65a6b68b3bcb4ae9e62e204939e6d2b8d430c/docs/custom_tables.md). + +### Best Practices +- Keep test data minimal but representative +- Use descriptive names for test tables and columns +- Document any assumptions about data format +- Include both positive and negative test cases +- Clean up test data after tests complete +- Test both static and dynamic table scenarios +- Verify index performance with appropriate test data +- Include tests for connection handling + +## Response Behavior +- NEVER make code changes without explicit user request +- ONLY write code or make changes when specifically asked +- For questions, provide explanations and guidance without making changes +- When asked about code, explain the current implementation without modifying it +- If code changes are needed, wait for explicit request before proceeding +- Always explain proposed changes before implementing them +- Get confirmation before making any code modifications + +## Code Style Guidelines +- Follow Go best practices and idioms +- Use clear, descriptive variable and function names +- Include comments for complex logic +- Write comprehensive tests for new functionality +- Keep functions focused and single-purpose + +## Testing Guidelines +- Write table-driven tests for multiple scenarios +- Test both success and error cases +- Use descriptive test case names +- Include edge cases in test coverage +- Test with realistic data formats and schemas + +## Documentation Requirements +- Add comments for exported functions and types +- Include examples in documentation where appropriate +- Document any assumptions or limitations +- Keep documentation up to date with code changes + +## Error Handling +- Return meaningful errors with context +- Use custom error types when appropriate +- Handle edge cases gracefully +- Log errors with sufficient detail for debugging + +## Performance Considerations +- Optimize for large datasets +- Minimize memory allocations +- Consider streaming for large files +- Profile code for performance bottlenecks + +## Security Guidelines +- Never expose sensitive data in logs or errors +- Validate all input data +- Use secure defaults +- Follow security best practices for data handling + +## Response Format +- Provide clear, concise explanations +- Include code examples when relevant +- Format responses in markdown +- Use backticks for code references +- Break down complex solutions into steps + +## File Organization +- Keep related code together +- Use appropriate package structure +- Follow Go project layout conventions +- Maintain clear separation of concerns + +## Version Control +- Write clear commit messages +- Keep commits focused and atomic +- Update tests with code changes +- Document breaking changes + +## Review Process +- Self-review code before submitting +- Ensure all tests pass +- Check for linting issues +- Verify documentation is complete diff --git a/.github/workflows/01-tailpipe-release.yaml b/.github/workflows/01-tailpipe-release.yaml index 21ddb842..4345a0fd 100644 --- a/.github/workflows/01-tailpipe-release.yaml +++ b/.github/workflows/01-tailpipe-release.yaml @@ -5,12 +5,12 @@ on: inputs: environment: type: choice - description: 'Select Release Type' + description: "Select Release Type" options: - # to change the values in this option, we also need to update the condition test below in at least 3 location. Search for github.event.inputs.environment - - Development (alpha) - - Development (beta) - - Final (RC and final release) + # to change the values in this option, we also need to update the condition test below in at least 3 location. Search for github.event.inputs.environment + - Development (alpha) + - Development (beta) + - Final (RC and final release) required: true version: description: "Version (without 'v')" @@ -38,13 +38,13 @@ jobs: - name: Parse semver string id: semver_parser - uses: booxmedialtd/ws-action-parse-semver@v1 + uses: booxmedialtd/ws-action-parse-semver@3576f3a20a39f8752fe0d8195f5ed384090285dc # v1.3.0 with: input_string: ${{ github.event.inputs.version }} - name: Checkout if: steps.semver_parser.outputs.prerelease == '' - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: turbot/homebrew-tap token: ${{ secrets.GH_ACCESS_TOKEN }} @@ -67,7 +67,7 @@ jobs: build_and_release: name: Build and Release Tailpipe needs: [ensure_branch_in_homebrew] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm steps: - name: validate if: github.ref == 'refs/heads/develop' @@ -82,20 +82,20 @@ jobs: fi - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: tailpipe ref: ${{ github.event.ref }} - name: Checkout Pipe Fittings Components repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: turbot/pipe-fittings path: pipe-fittings ref: develop - name: Checkout Tailpipe plugin SDK repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: turbot/tailpipe-plugin-sdk path: tailpipe-plugin-sdk @@ -103,13 +103,27 @@ jobs: ref: develop - name: Checkout Tailpipe Core Plugin repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: turbot/tailpipe-plugin-core path: tailpipe-plugin-core token: ${{ secrets.GH_ACCESS_TOKEN }} ref: main + - name: Set up Docker + uses: docker/setup-buildx-action@v3 + + - name: Install Docker (if needed) + run: | + if ! command -v docker &> /dev/null; then + sudo apt-get update + sudo apt-get install -y docker.io + fi + + - name: Verify Docker installation + run: | + docker --version + - name: Calculate version id: calculate_version run: | @@ -130,9 +144,9 @@ jobs: git push origin $VERSION # this is required, check golangci-lint-action docs - - uses: actions/setup-go@v4 + - uses: actions/setup-go@19bb51245e9c80abacb2e91cc42b33fa478b8639 # v4.2.1 with: - go-version: '1.23' + go-version: "1.23" cache: false # setup-go v4 caches by default, do not change this parameter, check golangci-lint-action doc: https://github.com/golangci/golangci-lint-action/pull/704 - name: Setup release environment @@ -143,6 +157,8 @@ jobs: - name: Release publish run: |- cd tailpipe + git config --global user.name "Tailpipe GitHub Actions Bot" + git config --global user.email noreply@github.com make release create_pr_in_homebrew: @@ -158,13 +174,13 @@ jobs: - name: Parse semver string id: semver_parser - uses: booxmedialtd/ws-action-parse-semver@v1 + uses: booxmedialtd/ws-action-parse-semver@3576f3a20a39f8752fe0d8195f5ed384090285dc # v1.3.0 with: input_string: ${{ github.event.inputs.version }} - name: Checkout if: steps.semver_parser.outputs.prerelease == '' - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: turbot/homebrew-tap token: ${{ secrets.GH_ACCESS_TOKEN }} @@ -202,13 +218,13 @@ jobs: - name: Parse semver string id: semver_parser - uses: booxmedialtd/ws-action-parse-semver@v1 + uses: booxmedialtd/ws-action-parse-semver@3576f3a20a39f8752fe0d8195f5ed384090285dc # v1.3.0 with: input_string: ${{ github.event.inputs.version }} - name: Checkout if: steps.semver_parser.outputs.prerelease == '' - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: turbot/homebrew-tap token: ${{ secrets.GH_ACCESS_TOKEN }} @@ -223,3 +239,117 @@ jobs: git add . git commit -m "Versioning brew formulas" git push origin $VERSION + + update_homebrew_tap: + name: Update homebrew-tap formula + if: ${{ github.event.inputs.environment == 'Final (RC and final release)' }} + needs: update_pr_for_versioning + runs-on: ubuntu-latest + steps: + - name: Calculate version + id: calculate_version + run: | + echo "VERSION=v${{ github.event.inputs.version }}" >> $GITHUB_ENV + + - name: Parse semver string + id: semver_parser + uses: booxmedialtd/ws-action-parse-semver@7784200024d6b3fc01253e617ec0168daf603de3 # v1.4.7 + with: + input_string: ${{ github.event.inputs.version }} + + - name: Checkout + if: steps.semver_parser.outputs.prerelease == '' + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: turbot/homebrew-tap + token: ${{ secrets.GH_ACCESS_TOKEN }} + ref: main + + - name: Get pull request title + if: steps.semver_parser.outputs.prerelease == '' + id: pr_title + run: >- + echo "PR_TITLE=$( + gh pr view $VERSION --json title | jq .title | tr -d '"' + )" >> $GITHUB_OUTPUT + + - name: Output + if: steps.semver_parser.outputs.prerelease == '' + run: | + echo ${{ steps.pr_title.outputs.PR_TITLE }} + echo ${{ env.VERSION }} + + - name: Fail if PR title does not match with version + if: steps.semver_parser.outputs.prerelease == '' + run: | + if [[ "${{ steps.pr_title.outputs.PR_TITLE }}" == "Tailpipe ${{ env.VERSION }}" ]]; then + echo "Correct version" + else + echo "Incorrect version" + exit 1 + fi + + - name: Merge pull request to update brew formula + if: steps.semver_parser.outputs.prerelease == '' + run: | + git fetch --all + gh pr merge $VERSION --squash --delete-branch + git push origin --delete bump-brew + + trigger_smoke_tests: + name: Trigger smoke tests + if: ${{ github.event.inputs.environment == 'Final (RC and final release)' }} + needs: [update_homebrew_tap] + runs-on: ubuntu-latest + steps: + - name: Calculate version + id: calculate_version + run: | + echo "VERSION=v${{ github.event.inputs.version }}" >> $GITHUB_ENV + + - name: Parse semver string + id: semver_parser + uses: booxmedialtd/ws-action-parse-semver@3576f3a20a39f8752fe0d8195f5ed384090285dc # v1.3.0 + with: + input_string: ${{ github.event.inputs.version }} + + - name: Trigger smoke test workflow + if: steps.semver_parser.outputs.prerelease == '' + run: | + echo "Triggering smoke test workflow for version $VERSION..." + gh workflow run "12-test-post-release-linux-distros.yaml" \ + --ref ${{ github.ref }} \ + --field version=$VERSION \ + --repo ${{ github.repository }} + env: + GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} + + - name: Get smoke test workflow run URL + if: steps.semver_parser.outputs.prerelease == '' + run: | + echo "Waiting for smoke test workflow to start..." + sleep 10 + + # Get the most recent run of the smoke test workflow + RUN_ID=$(gh run list \ + --workflow="12-test-post-release-linux-distros.yaml" \ + --repo ${{ github.repository }} \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId') + + if [ -n "$RUN_ID" ]; then + WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/$RUN_ID" + echo "āœ… Smoke test workflow triggered successfully!" + echo "šŸ”— Monitor progress at: $WORKFLOW_URL" + echo "" + echo "Workflow details:" + echo " - Version: $VERSION" + echo " - Workflow: 12-test-post-release-linux-distros.yaml" + echo " - Run ID: $RUN_ID" + else + echo "āš ļø Could not retrieve workflow run ID. Check manually at:" + echo "https://github.com/${{ github.repository }}/actions/workflows/12-test-post-release-linux-distros.yaml" + fi + env: + GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/10-test-lint.yaml b/.github/workflows/10-test-lint.yaml index bdfa2c05..049ba71e 100644 --- a/.github/workflows/10-test-lint.yaml +++ b/.github/workflows/10-test-lint.yaml @@ -15,41 +15,39 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Tailpipe repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: tailpipe - name: Checkout Pipe Fittings Components repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: turbot/pipe-fittings path: pipe-fittings ref: develop - name: Checkout Tailpipe plugin SDK repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: turbot/tailpipe-plugin-sdk path: tailpipe-plugin-sdk - token: ${{ secrets.GH_ACCESS_TOKEN }} ref: develop - name: Checkout Tailpipe Core Plugin repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: turbot/tailpipe-plugin-core path: tailpipe-plugin-core - token: ${{ secrets.GH_ACCESS_TOKEN }} ref: main # this is required, check golangci-lint-action docs - - uses: actions/setup-go@v5 + - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: go-version: '1.23' cache: false # setup-go v4 caches by default, do not change this parameter, check golangci-lint-action doc: https://github.com/golangci/golangci-lint-action/pull/704 - name: golangci-lint - uses: golangci/golangci-lint-action@v6 + uses: golangci/golangci-lint-action@55c2c1448f86e01eaae002a5a3a9624417608d84 # v6.5.2 with: version: latest args: --timeout=10m diff --git a/.github/workflows/11-test-acceptance.yaml b/.github/workflows/11-test-acceptance.yaml index 616a46b9..cbcbc901 100644 --- a/.github/workflows/11-test-acceptance.yaml +++ b/.github/workflows/11-test-acceptance.yaml @@ -16,39 +16,37 @@ env: jobs: goreleaser: name: Build - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: tailpipe ref: ${{ github.event.ref }} - name: Checkout Pipe Fittings Components repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: turbot/pipe-fittings path: pipe-fittings ref: develop - name: Checkout Tailpipe plugin SDK repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: turbot/tailpipe-plugin-sdk path: tailpipe-plugin-sdk - token: ${{ secrets.GH_ACCESS_TOKEN }} ref: develop - name: Checkout Tailpipe Core Plugin repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: turbot/tailpipe-plugin-core path: tailpipe-plugin-core - token: ${{ secrets.GH_ACCESS_TOKEN }} ref: main # this is required, check golangci-lint-action docs - - uses: actions/setup-go@v5 + - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: go-version: '1.23' cache: false # setup-go v4 caches by default, do not change this parameter, check golangci-lint-action doc: https://github.com/golangci/golangci-lint-action/pull/704 @@ -75,7 +73,7 @@ jobs: run: ls -l ~/artifacts - name: Save Linux Build Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: build-artifact-linux path: ~/artifacts/linux.tar.gz @@ -92,16 +90,23 @@ jobs: - "all_column_types" - "from_and_to" - "introspection" + - "partition_tests" + - "file_source" + - "partition_delete" + - "core_formats" + - "table_block" + - "config_precedence" + - "plugin" runs-on: ${{ matrix.platform }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: true path: tailpipe ref: ${{ github.event.ref }} - - uses: actions/setup-go@v5 + - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: go-version: 1.22 cache: false @@ -112,7 +117,7 @@ jobs: mkdir ~/artifacts - name: Download Linux Build Artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 if: ${{ matrix.platform == 'ubuntu-latest' }} with: name: build-artifact-linux @@ -134,7 +139,7 @@ jobs: - name: Install Tailpipe and plugins run: | - tailpipe plugin install chaos + tailpipe plugin install chaos aws - name: Run Test Suite id: run-test-suite @@ -171,13 +176,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Clean up Linux Build - uses: geekyeggo/delete-artifact@v5 + uses: geekyeggo/delete-artifact@f275313e70c08f6120db482d7a6b98377786765b # v5.1.0 with: name: build-artifact-linux failOnError: true - name: Clean up Darwin Build - uses: geekyeggo/delete-artifact@v5 + uses: geekyeggo/delete-artifact@f275313e70c08f6120db482d7a6b98377786765b # v5.1.0 with: name: build-artifact-darwin failOnError: true diff --git a/.github/workflows/12-test-post-release-linux-distros.yaml b/.github/workflows/12-test-post-release-linux-distros.yaml new file mode 100644 index 00000000..4e58287a --- /dev/null +++ b/.github/workflows/12-test-post-release-linux-distros.yaml @@ -0,0 +1,226 @@ +name: "12 - Test: Linux Distros (Post-release)" + +on: + workflow_dispatch: + inputs: + version: + description: "Version to test (with 'v' prefix, e.g., v1.0.0)" + required: true + type: string + +env: + # Version from input + VERSION: ${{ github.event.inputs.version }} + # Disable update checks during smoke tests + TAILPIPE_UPDATE_CHECK: false + SLACK_WEBHOOK_URL: ${{ secrets.PIPELING_RELEASE_BOT_WEBHOOK_URL }} + +jobs: + smoke_test_ubuntu_24: + name: Smoke test (Ubuntu 24, x86_64) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Download Linux Release Artifact + run: | + mkdir -p ./artifacts + gh release download ${{ env.VERSION }} \ + --pattern "tailpipe.linux.amd64.tar.gz" \ + --dir ./artifacts \ + --repo ${{ github.repository }} + # Rename to expected format + mv ./artifacts/tailpipe.linux.amd64.tar.gz ./artifacts/linux.tar.gz + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + + - name: Pull Ubuntu latest Image + run: docker pull ubuntu:latest + + - name: Create and Start Ubuntu latest Container + run: | + docker run -d --name ubuntu-24-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts ubuntu:latest tail -f /dev/null + + - name: Get runner/container info + run: | + docker exec ubuntu-24-test /scripts/linux_container_info.sh + + - name: Install dependencies and setup environment + run: | + docker exec ubuntu-24-test /scripts/prepare_ubuntu_container.sh + + - name: Run smoke tests + run: | + docker exec ubuntu-24-test /scripts/smoke_test.sh + + - name: Stop and Remove Container + run: | + docker stop ubuntu-24-test + docker rm ubuntu-24-test + + smoke_test_centos_9: + name: Smoke test (CentOS Stream 9, x86_64) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Download Linux Release Artifact + run: | + mkdir -p ./artifacts + gh release download ${{ env.VERSION }} \ + --pattern "tailpipe.linux.amd64.tar.gz" \ + --dir ./artifacts \ + --repo ${{ github.repository }} + # Rename to expected format + mv ./artifacts/tailpipe.linux.amd64.tar.gz ./artifacts/linux.tar.gz + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + + - name: Pull CentOS Stream 9 image + run: docker pull quay.io/centos/centos:stream9 + + - name: Create and Start CentOS stream9 Container + run: | + docker run -d --name centos-stream9-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts quay.io/centos/centos:stream9 tail -f /dev/null + + - name: Get runner/container info + run: | + docker exec centos-stream9-test /scripts/linux_container_info.sh + + - name: Install dependencies and setup environment + run: | + docker exec centos-stream9-test /scripts/prepare_centos_container.sh + + - name: Run smoke tests + run: | + docker exec centos-stream9-test /scripts/smoke_test.sh + + - name: Stop and Remove Container + run: | + docker stop centos-stream9-test + docker rm centos-stream9-test + + smoke_test_amazonlinux: + name: Smoke test (Amazon Linux 2023, x86_64) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Download Linux Release Artifact + run: | + mkdir -p ./artifacts + gh release download ${{ env.VERSION }} \ + --pattern "tailpipe.linux.amd64.tar.gz" \ + --dir ./artifacts \ + --repo ${{ github.repository }} + # Rename to expected format + mv ./artifacts/tailpipe.linux.amd64.tar.gz ./artifacts/linux.tar.gz + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + + - name: Pull Amazon Linux 2023 Image + run: docker pull amazonlinux:2023 + + - name: Create and Start Amazon Linux 2023 Container + run: | + docker run -d --name amazonlinux-2023-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts amazonlinux:2023 tail -f /dev/null + + - name: Get runner/container info + run: | + docker exec amazonlinux-2023-test /scripts/linux_container_info.sh + + - name: Install dependencies and setup environment + run: | + docker exec amazonlinux-2023-test /scripts/prepare_amazonlinux_container.sh + + - name: Run smoke tests + run: | + docker exec amazonlinux-2023-test /scripts/smoke_test.sh + + - name: Stop and Remove Container + run: | + docker stop amazonlinux-2023-test + docker rm amazonlinux-2023-test + + smoke_test_linux_arm64: + name: Smoke test (Ubuntu 24, ARM64) + runs-on: ubuntu-24.04-arm + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Download Linux ARM64 Release Artifact + run: | + mkdir -p ./artifacts + gh release download ${{ env.VERSION }} \ + --pattern "tailpipe.linux.arm64.tar.gz" \ + --dir ./artifacts \ + --repo ${{ github.repository }} + # Rename to expected format + mv ./artifacts/tailpipe.linux.arm64.tar.gz ./artifacts/linux.tar.gz + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Linux Artifacts and Install Binary + run: | + sudo tar -xzf ./artifacts/linux.tar.gz -C /usr/local/bin + sudo chmod +x /usr/local/bin/tailpipe + + - name: Install jq + run: | + sudo apt-get update + sudo apt-get install -y jq + + - name: Get runner/container info + run: | + uname -a + cat /etc/os-release + + - name: Run smoke tests + run: | + chmod +x ${{ github.workspace }}/scripts/smoke_test.sh + ${{ github.workspace }}/scripts/smoke_test.sh + + notify_completion: + name: Notify completion + runs-on: ubuntu-latest + needs: + [ + smoke_test_ubuntu_24, + smoke_test_centos_9, + smoke_test_amazonlinux, + smoke_test_linux_arm64, + ] + if: always() + steps: + - name: Check results and notify + run: | + # Check if all jobs succeeded + UBUNTU_24_RESULT="${{ needs.smoke_test_ubuntu_24.result }}" + CENTOS_9_RESULT="${{ needs.smoke_test_centos_9.result }}" + AMAZONLINUX_RESULT="${{ needs.smoke_test_amazonlinux.result }}" + ARM64_RESULT="${{ needs.smoke_test_linux_arm64.result }}" + + WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + if [ "$UBUNTU_24_RESULT" = "success" ] && [ "$CENTOS_9_RESULT" = "success" ] && [ "$AMAZONLINUX_RESULT" = "success" ] && [ "$ARM64_RESULT" = "success" ]; then + MESSAGE="āœ… Tailpipe ${{ env.VERSION }} smoke tests passed!\n\nšŸ”— View details: $WORKFLOW_URL" + else + MESSAGE="āŒ Tailpipe ${{ env.VERSION }} smoke tests failed!\n\nšŸ”— View details: $WORKFLOW_URL" + fi + + curl -X POST -H 'Content-type: application/json' \ + --data "{\"text\":\"$MESSAGE\"}" \ + ${{ env.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/30-stale.yaml b/.github/workflows/30-stale.yaml new file mode 100644 index 00000000..36e32042 --- /dev/null +++ b/.github/workflows/30-stale.yaml @@ -0,0 +1,43 @@ +name: "30 - Admin: Stale Issues and PRs" +on: + schedule: + - cron: "0 8 * * *" + workflow_dispatch: + inputs: + dryRun: + description: Set to true for a dry run + required: false + default: "false" + type: string + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - name: Stale issues and PRs + id: stale-issues-and-prs + uses: actions/stale@1160a2240286f5da8ec72b1c0816ce2481aabf84 # v8.0.0 + with: + # TODO: Add back the closing of stale issue part later on + # close-issue-message: | + # This issue was closed because it has been stalled for 90 days with no activity. + # close-issue-reason: "not_planned" + # close-pr-message: | + # This PR was closed because it has been stalled for 90 days with no activity. + # # Set days-before-close to 30 because we want to close the issue/PR after 90 days total, since days-before-stale is set to 60 + # days-before-close: 30 + days-before-close: -1 + days-before-stale: 60 + debug-only: ${{ inputs.dryRun }} + exempt-issue-labels: "good first issue,help wanted,blocker" + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-label: "stale" + stale-issue-message: | + This issue is stale because it has been open 60 days with no activity. + # This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 30 days. + stale-pr-label: "stale" + stale-pr-message: | + This PR is stale because it has been open 60 days with no activity. + # This PR is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 30 days. + start-date: "2021-02-09" + operations-per-run: 1000 diff --git a/.github/workflows/31-add-issues-to-pipeling-issue-tracker.yaml b/.github/workflows/31-add-issues-to-pipeling-issue-tracker.yaml new file mode 100644 index 00000000..594defea --- /dev/null +++ b/.github/workflows/31-add-issues-to-pipeling-issue-tracker.yaml @@ -0,0 +1,13 @@ +name: Assign Issue to Project + +on: + issues: + types: [opened] + +jobs: + add-to-project: + uses: turbot/steampipe-workflows/.github/workflows/assign-issue-to-pipeling-issue-tracker.yml@main + with: + issue_number: ${{ github.event.issue.number }} + repository: ${{ github.repository }} + secrets: inherit \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6f1db318..e16ec82a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,8 @@ *.dll *.so *.dylib - +/test_apps/ +/memtest # Editor cache and lock files *.swp *.swo @@ -28,4 +29,7 @@ go.work # Dist directory is created by goreleaser -/dist \ No newline at end of file +/dist + +# Sysroot directory is created by make build-sysroot +/sysroot \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index b6748a19..f42ba77e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -70,3 +70,4 @@ run: issues: exclude-dirs: - "tests/acceptance" + - "test_apps" diff --git a/.goreleaser.yml b/.goreleaser.yml index d8e8a667..f8834287 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,40 +1,50 @@ version: 2 builds: - - id: tailpipe-linux-arm64 + - id: tailpipe-linux-amd64 binary: tailpipe goos: - linux goarch: - - arm64 + - amd64 env: - - CC=aarch64-linux-gnu-gcc - - CXX=aarch64-linux-gnu-g++ + - CC=x86_64-linux-gnu-gcc + - CXX=x86_64-linux-gnu-g++ + - CGO_ENABLED=1 + - GOFLAGS= + - CGO_LDFLAGS= # Custom ldflags. # # Default: '-s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} -X main.builtBy=goreleaser' # Templates: allowed ldflags: - # Go Releaser analyzes your Git repository and identifies the most recent Git tag (typically the highest version number) as the version for your release. + # Goreleaser analyzes your Git repository and identifies the most recent Git tag (typically the highest version number) as the version for your release. # This is how it determines the value of {{.Version}}. - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser - - id: tailpipe-linux-amd64 + tags: [] + + - id: tailpipe-linux-arm64 binary: tailpipe goos: - linux goarch: - - amd64 + - arm64 env: - - CC=x86_64-linux-gnu-gcc - - CXX=x86_64-linux-gnu-g++ + - CC=gcc + - CXX=g++ + - CGO_ENABLED=1 + - GOFLAGS= + - CGO_LDFLAGS= ldflags: - -s -w -X main.version={{.Version}} -X main.date={{.Date}} -X main.commit={{.Commit}} -X main.builtBy=goreleaser + tags: [] + - id: tailpipe-darwin-arm64 binary: tailpipe goos: diff --git a/20250413_CustomTableTesting.md b/20250413_CustomTableTesting.md deleted file mode 100644 index 35b0cddd..00000000 --- a/20250413_CustomTableTesting.md +++ /dev/null @@ -1,1352 +0,0 @@ -# 20250413 Custom Table Testing - -## Row Enrichment - -### Basic Test - Custom Regex Format [Core Plugin Only] - -Basic minimal test - -Config -```hcl -table "t" { - column "tp_timestamp" { - source = "time_local" - } -} - -format "regex" "r" { - layout = `^(?P[^ ]*) - (?P[^ ]*) \[(?P[^\]]*)\] "(?P\S+)(?: +(?P[^ ]+))?(?: +(?P\S+))?" (?P[^ ]*) (?P[^ ]*) "(?P.*?)" "(?P.*?)"` -} - -partition "t" "p" { - source "file" { - format = format.regex.r - paths = ["/Users/graza/tailpipe_data/nginx_access_logs"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -Query -``` -> select tp_timestamp, body_bytes_sent, http_referer, http_user_agent, remote_addr, remote_user, request_method, request_uri, server_protocol, status, time_local from t limit 5; -+---------------------+-----------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------+----------------+-----------------------------------------------------------------+-----------------+--------+----------------------------+ -| tp_timestamp | body_bytes_sent | http_referer | http_user_agent | remote_addr | remote_user | request_method | request_uri | server_protocol | status | time_local | -+---------------------+-----------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------+----------------+-----------------------------------------------------------------+-----------------+--------+----------------------------+ -| 2024-07-29 10:30:20 | 3041 | - | Opera/9.57 (Macintosh; U; Intel Mac OS X 10_8_5; en-US) Presto/2.8.308 Version/10.00 | 4.73.13.210 | - | DELETE | /capability/Pre-emptive/Distributed/hybrid/actuating.css | HTTP/1.1 | 200 | 29/Jul/2024:11:30:20 +0100 | -| 2024-07-29 10:30:21 | 2764 | - | Mozilla/5.0 (iPad; CPU OS 9_2_2 like Mac OS X; en-US) AppleWebKit/536.18.6 (KHTML, like Gecko) Version/3.0.5 Mobile/8B112 Safari/6536.18.6 | 120.201.143.102 | - | GET | /exuding%20Horizontal-paradigm-discrete_parallelism.htm | HTTP/1.1 | 200 | 29/Jul/2024:11:30:21 +0100 | -| 2024-07-29 10:30:22 | 45 | - | Mozilla/5.0 (Windows NT 5.01; en-US; rv:1.9.0.20) Gecko/1933-06-04 Firefox/35.0 | 181.214.126.176 | - | PUT | /client-server/Persevering/hybrid-analyzing/standardization.css | HTTP/1.1 | 404 | 29/Jul/2024:11:30:22 +0100 | -| 2024-07-29 10:30:23 | 1176 | - | Opera/9.60 (X11; Linux x86_64; en-US) Presto/2.8.280 Version/11.00 | 42.72.2.165 | - | PATCH | /Automated/asynchronous/knowledge%20base.js | HTTP/1.1 | 200 | 29/Jul/2024:11:30:23 +0100 | -| 2024-07-29 10:30:24 | 1035 | - | Mozilla/5.0 (X11; Linux x86_64; rv:6.0) Gecko/1913-11-04 Firefox/36.0 | 80.164.243.11 | - | GET | /next%20generation-knowledge%20user/encryption_Sharable.js | HTTP/1.1 | 200 | 29/Jul/2024:11:30:24 +0100 | -+---------------------+-----------------+--------------+--------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------+----------------+-----------------------------------------------------------------+-----------------+--------+----------------------------+ -``` - -### Basic Test - Nginx Preset Format [Core/Nginx Cross Plugin] - -Config -```hcl -table "t" { - column "tp_timestamp" { - source = "time_local" - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.default - paths = ["/Users/graza/tailpipe_data/nginx_access_logs"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -Query -``` -> select tp_timestamp, body_bytes_sent, http_referer, http_user_agent, remote_addr, remote_user, request_method, request_uri, server_protocol, status, time_local from t limit 5; -+---------------------+-----------------+--------------+-----------------------------------------------------------------------------------------------------------------+-----------------+-------------+----------------+---------------------------------------------------------+-----------------+--------+----------------------------+ -| tp_timestamp | body_bytes_sent | http_referer | http_user_agent | remote_addr | remote_user | request_method | request_uri | server_protocol | status | time_local | -+---------------------+-----------------+--------------+-----------------------------------------------------------------------------------------------------------------+-----------------+-------------+----------------+---------------------------------------------------------+-----------------+--------+----------------------------+ -| 2024-07-29 10:05:41 | 43 | - | Mozilla/5.0 (Windows 98; Win 9x 4.90) AppleWebKit/5310 (KHTML, like Gecko) Chrome/38.0.801.0 Mobile Safari/5310 | 141.25.76.110 | - | GET | /array/core%20knowledge%20user/Cross-group.gif | HTTP/1.1 | 301 | 29/Jul/2024:11:05:41 +0100 | -| 2024-07-29 10:05:42 | 1401 | - | Mozilla/5.0 (Windows NT 5.0) AppleWebKit/5331 (KHTML, like Gecko) Chrome/36.0.868.0 Mobile Safari/5331 | 69.181.196.125 | - | GET | /structure-Managed%20project/middleware-application.htm | HTTP/1.1 | 200 | 29/Jul/2024:11:05:42 +0100 | -| 2024-07-29 10:05:43 | 74 | - | Mozilla/5.0 (Windows NT 5.01; en-US; rv:1.9.1.20) Gecko/1946-10-03 Firefox/37.0 | 50.247.11.252 | - | DELETE | /flexibility-Persevering/mission-critical/Monitored.js | HTTP/1.1 | 400 | 29/Jul/2024:11:05:43 +0100 | -| 2024-07-29 10:05:44 | 2877 | - | Mozilla/5.0 (X11; Linux i686; rv:7.0) Gecko/1959-02-09 Firefox/36.0 | 101.223.247.121 | - | POST | /grid-enabled/capacity-Persevering/radical.hmtl | HTTP/1.1 | 200 | 29/Jul/2024:11:05:44 +0100 | -| 2024-07-29 10:05:45 | 2563 | - | Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_6_10 rv:2.0) Gecko/1924-25-04 Firefox/36.0 | 168.10.143.27 | - | GET | /Persevering%20Open-architected_national/Horizontal.js | HTTP/1.1 | 200 | 29/Jul/2024:11:05:45 +0100 | -+---------------------+-----------------+--------------+-----------------------------------------------------------------------------------------------------------------+-----------------+-------------+----------------+---------------------------------------------------------+-----------------+--------+----------------------------+ -``` - -### Basic Test - Nginx Custom Format [Core/Nginx Cross Plugin] - -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - column "tp_timestamp" { - source = "time_local" - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -Query -``` -> SELECT tp_timestamp, body_bytes_sent, http_referer, http_user_agent, remote_addr, remote_user, request_method, request_uri, server_protocol, status, time_local, request_time FROM t LIMIT 5; -+---------------------+-----------------+--------------+---------------------------------------------------------------------------------------------------------------------------+-----------------+-------------+----------------+-----------------+-----------------+--------+----------------------------+--------------+ -| tp_timestamp | body_bytes_sent | http_referer | http_user_agent | remote_addr | remote_user | request_method | request_uri | server_protocol | status | time_local | request_time | -+---------------------+-----------------+--------------+---------------------------------------------------------------------------------------------------------------------------+-----------------+-------------+----------------+-----------------+-----------------+--------+----------------------------+--------------+ -| 2025-03-30 09:37:55 | 915756 | - | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 | 174.132.116.92 | - | GET | /api/v1/payment | HTTP/1.1 | 204 | 30/Mar/2025:10:37:55 +0100 | 0.327 | -| 2025-03-30 09:37:56 | 845190 | - | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 | 243.52.146.253 | - | GET | /wp-admin | HTTP/1.1 | 301 | 30/Mar/2025:10:37:56 +0100 | 3.906 | -| 2025-03-30 09:37:56 | 469267 | - | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 | 121.55.217.233 | - | DELETE | /metrics | HTTP/1.1 | 204 | 30/Mar/2025:10:37:56 +0100 | 0.328 | -| 2025-03-30 09:37:57 | 174142 | - | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 | 192.168.25.127 | - | GET | / | HTTP/1.1 | 200 | 30/Mar/2025:10:37:57 +0100 | 0.241 | -| 2025-03-30 09:37:58 | 465818 | - | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 | 102.201.100.205 | - | DELETE | /api/v1/orders | HTTP/1.1 | 301 | 30/Mar/2025:10:37:58 +0100 | 5.592 | -``` - -### Custom Table - Add Description - -Config -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - description = "nginx access log" - - column "tp_timestamp" { - source = "time_local" - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -```sh -āÆ tailpipe table list -NAME PLUGIN LOCAL SIZE FILES ROWS DESCRIPTION -apache_access_log hub.tailpipe.io/plugins/turbot/apache@latest - - - -nginx_access_log hub.tailpipe.io/plugins/turbot/nginx@latest - - - -t hub.tailpipe.io/plugins/turbot/core@latest 7.5 MB 2 186,297 nginx access log -``` - -### NuffIf - Table Level - -Note: If all values are `null` the column isn't in the output dataset, for example setting to `-` removes the `http_referer` and `remote_user` columns - -Config -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - description = "nginx access log" - - null_if = "-" - - column "tp_timestamp" { - source = "time_local" - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -``` -> .inspect t -Column - Type -body_bytes_sent varchar -http_user_agent varchar -remote_addr varchar -request_method varchar -request_time varchar -request_uri varchar -server_protocol varchar -status varchar -time_local varchar -tp_ -``` - -If you set it to `200` you can null the `status` rows which have a success code: -Config -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - description = "nginx access log" - - null_if = "200" - - column "tp_timestamp" { - source = "time_local" - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` -``` -> select distinct status from t; -+--------+ -| status | -+--------+ -| 401 | -| | -| 404 | -| 500 | -| 201 | -| 204 | -| 302 | -| 403 | -| 400 | -| 301 | -+--------+ -``` - -### NuffIf - Column Level - -Removing 404 status from our rows - -Config: -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - description = "nginx access log" - - column "tp_timestamp" { - source = "time_local" - } - - column "status" { - source = "status" - null_if = "404" - type = "integer" - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -``` -> select distinct status from t; -+--------+ -| status | -+--------+ -| 200 | -| | -| 301 | -| 401 | -| 302 | -| 500 | -| 400 | -| 403 | -| 204 | -| 201 | -+--------+ -``` - -### Transform - Amend existing column - -Make request_method lowercase - -Config: -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - column "tp_timestamp" { - source = "time_local" - } - - column "request_method" { - transform = "lower(request_method)" - type = "varchar" - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -Query -``` -> select distinct request_method from t; -+----------------+ -| request_method | -+----------------+ -| put | -| post | -| get | -| delete | -+----------------+ -``` - -### Transform - Create a new column - -Add `raw_request` the 3 components of $request (which we separate) - -Config: -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - column "tp_timestamp" { - source = "time_local" - } - - column "raw_request" { - transform = "concat(request_method, ' ', request_uri, ' ', server_protocol)" - type = "varchar" - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -Query: -``` -> select distinct raw_request from t limit 5; -+--------------------------------------------------------------------+ -| raw_request | -+--------------------------------------------------------------------+ -| PUT /api/v1/orders HTTP/1.1 | -| GET /api/v1/internal/config?..%2F..%2F..%2Fetc%2Fpasswd HTTP/1.1 | -| GET /health?..%2F..%2F..%2Fetc%2Fpasswd HTTP/1.1 | -| PUT /api/v1/users?%3Cscript%3Ealert%281%29%3C%2Fscript%3E HTTP/1.1 | -| PUT /phpMyAdmin?%27+OR+%271%27%3D%271 HTTP/1.1 | -+--------------------------------------------------------------------+ -``` - -### Required Column Not Existing - Expect Errors - -Config: -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - column "tp_timestamp" { - source = "time_local" - } - - column "not_here" { - required = true - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -Output: -```sh -āÆ tailpipe collect t.p --from T-10Y - -Collecting logs for t.p from 2015-04-13 - -Artifacts: - Discovered: 2 - Downloaded: 2 3.9MB - Extracted: 2 - -Rows: - Received: 186,297 - Enriched: 0 - Saved: 0 - Errors: 186,297 - -Files: - No files to compact. - -Errors: - 20250330_access.log.gz: 99,122 rows have missing fields: not_here - 20250331_access.log.gz: 87,175 rows have missing fields: not_here - Set TAILPIPE_LOG_LEVEL=ERROR for details. - -``` - -### MapFields - Obtain subset of data - -Get only tp_fields (enforced), along with status and request* fields - -Config: -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - column "tp_timestamp" { - source = "time_local" - } - - map_fields = ["status", "request*"] -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -``` -> .inspect t -Column Type -request_method varchar -request_time varchar -request_uri varchar -status varchar -tp_... -``` - -### MapFields - Given a field that doesn't exist - -Since this is a pattern match, this should allow us to obtain tp_fields and no additional data - -Config: -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - column "tp_timestamp" { - source = "time_local" - } - - map_fields = ["not_matching_pattern"] -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -``` -> .inspect t -Column Type -tp_akas varchar[] -tp_date date -tp_destination_ip varchar -tp_domains varchar[] -tp_emails varchar[] -tp_id varchar -tp_index varchar -tp_ingest_timestamp timestamp -tp_ips varchar[] -tp_partition varchar -tp_source_ip varchar -tp_source_location varchar -tp_source_name varchar -tp_source_type varchar -tp_table varchar -tp_tags varchar[] -tp_timestamp timestamp -tp_usernames varchar[] -``` - -### MapFields - Wildcard Testing `*a*` - -Config: -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - column "tp_timestamp" { - source = "time_local" - } - - map_fields = ["*a*"] -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -``` -> .inspect t -Column Type -http_user_agent varchar -remote_addr varchar -status varchar -time_local varchar -tp_... -``` - -### Transform - Non-Existent Function [Error Expected] - -Config: -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - column "tp_timestamp" { - source = "time_local" - } - - column "request_method" { - type = "varchar" - transform = "fake_function(request_method)" - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -``` -āÆ tailpipe collect t.p --from T-10Y - -Collecting logs for t.p from 2015-04-13 - -Artifacts: - Discovered: 2 - Downloaded: 2 3.9MB - Extracted: 2 - -Rows: - Received: 186,297 - Enriched: 186,297 - Saved: 0 - Errors: 186,297 - -Files: - No files to compact. - -Errors: - 1744553252666-0.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: mean "qufake_function(request_method) as "request_method", - ^ - 1744553252666-1.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: mean "qufake_function(request_method) as "request_method", - ^ - 1744553252666-2.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: mean "qufake_function(request_method) as "request_method", - ^ - 1744553252666-3.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: mean "qufake_function(request_method) as "request_method", - ^ - 1744553252666-4.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: mean "qufake_function(request_method) as "request_method", - ^ - 1744553252666-5.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: mean "qufake_function(request_method) as "request_method", - ^ - 1744553252666-6.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: mean "qufake_function(request_method) as "request_method", - ^ - 1744553252666-7.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: mean "qufake_function(request_method) as "request_method", - ^ - 1744553252666-8.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: mean "qufake_function(request_method) as "request_method", - ^ - 1744553252666-9.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: mean "qufake_function(request_method) as "request_method", - ^ - 1744553252666-10.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: mean "qufake_function(request_method) as "request_method", - ^ - 1744553252666-11.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: mean "qufake_function(request_method) as "request_method", - ^ - 1744553252666-12.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: mean "qufake_function(request_method) as "request_method", - ^ - 1744553252666-13.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: mean "qufake_function(request_method) as "request_method", - ^ - 1744553252666-14.jsonl: Catalog Error: Scalar Function with name fake_function does not exist! -Did you mean "quantile_cont"? - -LINE 4: fake_function(request_method) as "request_method", - ^ - Truncated. Set TAILPIPE_LOG_LEVEL=ERROR for details. - -Completed: 2s -``` - -### Transform - Same Field (essentially do nothing) - -Config: -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - column "tp_timestamp" { - source = "time_local" - } - - column "request_method" { - type = "varchar" - transform = "request_method" - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -``` -> select distinct request_method from t; -+----------------+ -| request_method | -+----------------+ -| PUT | -| POST | -| DELETE | -| GET | -+----------------+ -``` - -### Transform - Replication of "Source" - -Config: -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - column "tp_timestamp" { - source = "time_local" - } - - column "banana" { - type = "varchar" - transform = "request_method" - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -``` -> select distinct banana from t; -+--------+ -| banana | -+--------+ -| PUT | -| POST | -| GET | -| DELETE | -+--------+ -``` - -### Transform - Replication of "Source" but with non-existent source - -Config: -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - column "tp_timestamp" { - source = "time_local" - } - - column "banana" { - type = "varchar" - transform = "not_here" - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -``` -āÆ tailpipe collect t.p --from T-10Y - -Collecting logs for t.p from 2015-04-13 - -Artifacts: - Discovered: 2 - Downloaded: 2 3.9MB - Extracted: 2 - -Rows: - Received: 186,297 - Enriched: 186,297 - Saved: 0 - Errors: 186,297 - -Files: - No files to compact. - -Errors: - 1744553687330-0.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: e bindinnot_here as "banana", - ^ - 1744553687330-1.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: e bindinnot_here as "banana", - ^ - 1744553687330-2.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: e bindinnot_here as "banana", - ^ - 1744553687330-3.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: e bindinnot_here as "banana", - ^ - 1744553687330-4.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: e bindinnot_here as "banana", - ^ - 1744553687330-5.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: e bindinnot_here as "banana", - ^ - 1744553687330-6.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: e bindinnot_here as "banana", - ^ - 1744553687330-7.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: e bindinnot_here as "banana", - ^ - 1744553687330-8.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: e bindinnot_here as "banana", - ^ - 1744553687330-9.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: e bindinnot_here as "banana", - ^ - 1744553687330-10.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: e bindinnot_here as "banana", - ^ - 1744553687330-11.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: e bindinnot_here as "banana", - ^ - 1744553687330-12.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: e bindinnot_here as "banana", - ^ - 1744553687330-13.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: e bindinnot_here as "banana", - ^ - 1744553687330-14.jsonl: Binder Error: Referenced column "not_here" not found in FROM clause! -Candidate bindings: "tp_usernames", "http_user_agent", "remote_user", "remote_addr", "http_referer" - -LINE 4: not_here as "banana", - ^ - Truncated. Set TAILPIPE_LOG_LEVEL=ERROR for details. - -Completed: 2s -``` - -### Transform - strptime (since we removed time_format) - -Config: -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - column "tp_timestamp" { - source = "time_local" - } - - column "parsed_time" { - type = "timestamp" - transform = "strptime(time_local, '%d/%b/%Y:%H:%M:%S %z')" - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -``` -> .inspect t -Column Type -body_bytes_sent varchar -http_referer varchar -http_user_agent varchar -parsed_time timestamp with time zone -remote_addr varchar -remote_user varchar -request_method varchar -request_time varchar -request_uri varchar -server_protocol varchar -status varchar -time_local varchar -tp_... -``` - -> Additional Note: wasn't able to convert the type of time_local using this approach - -```hcl -format "nginx_access_log" "custom" { - layout = `$remote_addr - $remote_user [$time_local] "$request_method $request_uri $server_protocol" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time` -} - -table "t" { - column "tp_timestamp" { - source = "time_local" - } - - column "time_local" { - type = "timestamp" - transform = "strptime(time_local, '%d/%b/%Y:%H:%M:%S %z')" - } -} - -partition "t" "p" { - source "file" { - format = format.nginx_access_log.custom - paths = ["/Users/graza/tailpipe_data/nginx2"] - file_layout = `%{YEAR:year}%{MONTHNUM:month}%{MONTHDAY:day}_%{DATA}.log` - } -} -``` - -results in -``` - 1744554292192-0.jsonl: Binder Error: No function matches the given name and argument types 'strptime(TIMESTAMP, STRING_LITERAL)'. You might need to add explicit type casts. - Saved:Candidate functions: - Errorsstrptime(VARCHAR, VARCHAR) -> TIMESTAMP - strptime(VARCHAR, VARCHAR[]) -> TIMESTAMP - - -LINE 4: Candidatstrptime(time_local, '%d/%b/%Y:%H:%M:%S %z') as "time_local... - ^ -``` - -and omitting the type results in: -``` -āÆ tailpipe collect t.p --from T-10Y -Error: collection error: table 't' returned invalid schema: column type must be specified if column is optional (column 'time_local') -``` - - -### Additional Notes / Actions -- Need to enhance/tidy error messages when function used in transform doesn't exist -- Need to enhance/tidy error messages when column used in transform doesn't exist -- Trying to convert a VARCHAR column to TIMESTAMP with a STRPTIME transform results in errors. - -## JSONL Direct Conversion - -### Basic Test - -Minimal setup - -Config: -```hcl -table "t" { - column "tp_timestamp" { - source = "timestamp" - } -} - -partition "t" "p" { - source "file" { - format = format.jsonl.default - paths = ["/Users/graza/tailpipe_data/jsonl"] - file_layout = `.jsonl` - } -} -``` - -``` -> .inspect t -Column Type -ip_address varchar -method varchar -path varchar -protocol varchar -query varchar -request_headers struct(accept varchar, "authorization" varchar, "user-agent" varchar) -request_id varchar -request_time double -response_headers struct("cache-control" varchar, "content-type" varchar) -response_size bigint -status bigint -timestamp varchar -tp_akas varchar[] -tp_date date -tp_destination_ip varchar -tp_domains varchar[] -tp_emails varchar[] -tp_id varchar -tp_index varchar -tp_ingest_timestamp timestamp -tp_ips varchar[] -tp_partition varchar -tp_source_ip varchar -tp_source_location varchar -tp_source_name varchar -tp_source_type varchar -tp_table varchar -tp_tags varchar[] -tp_timestamp timestamp -tp_usernames varchar[] -user_agent varchar -``` - -It looks like we've lost nested values! -``` -+--------------------------------------------------------+ -| request_headers | -+--------------------------------------------------------+ -| map[accept: authorization: user-agent:] | -| map[accept: authorization: user-agent:] | -| map[accept: authorization: user-agent:] | -| map[accept: authorization: user-agent:] | -| map[accept: authorization: user-agent:] | -| map[accept: authorization: user-agent:] | -| map[accept: authorization: user-agent:] | -| map[accept: authorization: user-agent:] | -| map[accept: authorization: user-agent:] | -| map[accept: authorization: user-agent:] | -+--------------------------------------------------------+ -``` - -### Changing Struct to Json - -Config: -```hcl -table "t" { - column "tp_timestamp" { - source = "timestamp" - } - - column "request_headers" { - type = "json" - } - - column "response_headers" { - type = "json" - } -} - -partition "t" "p" { - source "file" { - format = format.jsonl.default - paths = ["/Users/graza/tailpipe_data/jsonl"] - file_layout = `.jsonl` - } -} -``` - -Now we have our nested values back! -``` -> select request_headers from t limit 10; -+----------------------------------------------------------------------------------------------------------------------+ -| request_headers | -+----------------------------------------------------------------------------------------------------------------------+ -| map[Accept:application/json Authorization:Bearer token123 User-Agent:okhttp/4.9.0] | -| map[Accept:application/json Authorization:Bearer token123 User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 13_0_1)] | -| map[Accept:application/json Authorization:Bearer token123 User-Agent:curl/8.0.1] | -| map[Accept:application/json Authorization:Bearer token123 User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 13_0_1)] | -| map[Accept:application/json Authorization:Bearer token123 User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 13_0_1)] | -| map[Accept:application/json Authorization:Bearer token123 User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64)] | -| map[Accept:application/json Authorization:Bearer token123 User-Agent:okhttp/4.9.0] | -| map[Accept:application/json Authorization:Bearer token123 User-Agent:PostmanRuntime/7.32.0] | -| map[Accept:application/json Authorization:Bearer token123 User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 13_0_1)] | -| map[Accept:application/json Authorization:Bearer token123 User-Agent:PostmanRuntime/7.32.0] | -+----------------------------------------------------------------------------------------------------------------------+ -``` - -### Complex Test - MapFields/Transforms/Etc - -Config: -```hcl -table "t" { - column "tp_timestamp" { - source = "timestamp" - } - - column "request_headers" { - type = "json" - } - - column "response_headers" { - type = "json" - } - - column "client" { - source = "ip_address" - type = "varchar" - } - - column "summary" { - transform = "client || ' attemped to ' || lower(method) || ' ' || path || coalesce(query,'')" - type = "varchar" - } - - map_fields = [ - "status" - ] -} - -partition "t" "p" { - source "file" { - format = format.jsonl.default - paths = ["/Users/graza/tailpipe_data/jsonl"] - file_layout = `.jsonl` - } -} -``` - -``` -> .inspect t -Column Type -client varchar -request_headers json -response_headers json -status bigint -summary varchar -tp_akas varchar[] -tp_date date -tp_destination_ip varchar -tp_domains varchar[] -tp_emails varchar[] -tp_id varchar -tp_index varchar -tp_ingest_timestamp timestamp -tp_ips varchar[] -tp_partition varchar -tp_source_ip varchar -tp_source_location varchar -tp_source_name varchar -tp_source_type varchar -tp_table varchar -tp_tags varchar[] -tp_timestamp timestamp -tp_usernames varchar[] -``` - -``` -> select summary from t limit 10; -+--------------------------------------------------+ -| summary | -+--------------------------------------------------+ -| 119.83.160.27 attemped to delete /api/orders | -| 98.59.183.181 attemped to put /dashboard | -| 113.136.102.108 attemped to delete /dashboard | -| 232.56.198.126 attemped to get /dashboard?page=1 | -| 173.209.26.227 attemped to put /api/orders | -| 218.32.250.216 attemped to get /logout | -| 226.71.3.13 attemped to get /logout?sort=asc | -| 68.1.173.68 attemped to delete /login | -| 128.10.66.112 attemped to delete /api/orders | -| 193.25.149.73 attemped to get /dashboard | -+--------------------------------------------------+ -``` - -### Additional Notes / Actions -- ERROR: Nested when automatically converted to struct fields have now been lower-cased and lost their values - - Can alleviate this by setting column to `json` type and then we keep the casing/values of nested objects - -## Delimited Direct Conversion - -### Basic Test - -Config: -```hcl -table "t" { - column "tp_timestamp" { - source = "BillingPeriodStartDate" - } -} - -partition "t" "p" { - source "file" { - format = format.delimited.default - paths = ["/Users/graza/tailpipe_data/billing"] - file_layout = `.csv` - } -} -``` - -Sh: -```sh -āÆ tailpipe collect t.p --from T-10Y - -Collecting logs for t.p from 2015-04-13 - -Artifacts: - Discovered: 35 - Downloaded: 35 223MB - Extracted: 35 - -Rows: - Received: 474,534 - Enriched: 474,534 - Saved: 449,590 - Errors: 24,944 - -Files: - Compacted: 33 => 33 - -Errors: - 174455638275-6.jsonl: inferred schema change detected - consider specifying a column type in table definition: 'InvoiceID': 'bigint' -> 'varchar' - 174455638275-34.jsonl: inferred schema change detected - consider specifying a column type in table definition: 'InvoiceID': 'bigint' -> 'varchar' - Set TAILPIPE_LOG_LEVEL=ERROR for details. - -Completed: 4s -``` - -Query: -``` -> select count(*) from t; -+--------------+ -| count_star() | -+--------------+ -| 449590 | -+--------------+ -> .inspect t -Column Type -BillingPeriodEndDate timestamp -BillingPeriodStartDate timestamp -BlendedRate json -CostBeforeTax double -Credits double -CurrencyCode varchar -InvoiceDate timestamp -InvoiceID bigint -ItemDescription varchar -LinkedAccountId varchar -LinkedAccountName varchar -Operation varchar -PayerAccountId bigint -PayerAccountName varchar -PayerPONumber json -ProductCode varchar -ProductName varchar -RateId bigint -RecordID varchar -RecordType varchar -SellerOfRecord varchar -TaxAmount double -TaxType varchar -TaxationAddress json -TotalCost double -UsageEndDate timestamp -UsageQuantity double -UsageStartDate timestamp -UsageType varchar -tp_akas varchar[] -tp_date date -tp_destination_ip varchar -tp_domains varchar[] -tp_emails varchar[] -tp_id varchar -tp_index varchar -tp_ingest_timestamp timestamp -tp_ips varchar[] -tp_partition varchar -tp_source_ip varchar -tp_source_location varchar -tp_source_name varchar -tp_source_type varchar -tp_table varchar -tp_tags varchar[] -tp_timestamp timestamp -tp_usernames varchar[] -``` - -### Changing Type - -InvoiceID can be omitted, or non-numeric so adjusted this to varchar - -Config: -```hcl -table "t" { - column "tp_timestamp" { - source = "BillingPeriodStartDate" - } - - column "InvoiceID" { - type = "varchar" - } -} - -partition "t" "p" { - source "file" { - format = format.delimited.default - paths = ["/Users/graza/tailpipe_data/billing"] - file_layout = `.csv` - } -} -``` - -``` -āÆ tailpipe collect t.p --from T-10Y - -Collecting logs for t.p from 2015-04-13 - -Artifacts: - Discovered: 35 - Downloaded: 35 223MB - Extracted: 35 - -Rows: - Received: 474,534 - Enriched: 474,534 - Saved: 474,533 - Errors: 1 - -Files: - Compacted: 35 => 35 - -Errors: - 1744556892361-34.jsonl: validation failed - found null values in columns: tp_timestamp, tp_date - Set TAILPIPE_LOG_LEVEL=ERROR for details. - -Completed: 4s -``` - diff --git a/CHANGELOG.md b/CHANGELOG.md index 67658c19..3508990a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,114 @@ -## v0.3.0 [2025-04-02] +## v0.7.1 [2025-10-07] +_Bug Fixes_ +- Build: Restored CentOS/RHEL 9 compatibility by pinning the build image to an older libstdc++/GCC baseline. Previous build linked against newer GLIBCXX symbols, causing Tailpipe to fail on CentOS/RHEL 9. + +## v0.7.0 [2025-09-22] + +### _Major Changes_ +* Replace native Parquet conversion with a **DuckLake database backend**. ([#546](https://github.com/turbot/tailpipe/issues/546)) + - DuckLake is DuckDBā€™s new lakehouse format: data remains in Parquet files, but metadata is efficiently tracked in a + separate DuckDB database. + - DuckLake supports function-based partitioning, which allows data to be partitioned by year and month. This enables + efficient file pruning on `tp_timestamp` without needing a separate `tp_date` filter. A `tp_date` column will still + be present for compatibility, but it is no longer required for efficient query filtering. + - Existing data will be **automatically migrated** the next time Tailpipe runs. Migration does **not** + occur if progress output is disabled (`--progress=false`) or when using machine-readable output (`json`, `line`, + `csv`). + + **Note:** For CentOS/RHEL users, the minimum supported version is now **CentOS Stream 10 / RHEL 10** due to `libstdc++` library compatibility. + +* The `connect` command now returns the path to an **initialisation SQL script** instead of the database path. ([#550](https://github.com/turbot/tailpipe/issues/550)) + - The script sets up DuckDB with required extensions, attaches the Tailpipe database, and defines views with optional + filters. + - You can pass the generated script to DuckDB using the `--init` argument to immediately configure the session. For + example: + ```sh + duckdb --init $(tailpipe connect) + ``` + **Note:** The minimum supported DuckDB version is 1.4.0. + +### _Bug Fixes_ +* Include partitions for local plugins in the `tailpipe plugin list` command. ([#538](https://github.com/turbot/tailpipe/issues/538)) + + +## v0.6.2 [2025-07-24] +_Bug fixes_ +* Fix issue where `--to` was not respected for zero granularity data. ([#483](https://github.com/turbot/tailpipe/issues/483)) +* Fix issue where the relative time passed to `from/to` args were getting parsed incorrectly. ([#485](https://github.com/turbot/tailpipe/issues/485)) +* Fix issue where Tailpipe was crashing if the collection state file had nil trunk states from the previous collection. ([#489](https://github.com/turbot/tailpipe/issues/489)) +* Fix `.inspect` output to show the plugin name for custom tables. ([#360](https://github.com/turbot/tailpipe/issues/360)) +* Fix query JSON outputs to be consistent with DuckDB. ([#432](https://github.com/turbot/tailpipe/issues/432)) + +_Dependencies_ +* Upgrade `go-viper/mapstructure/v2` and `oauth2` packages to remediate high and moderate vulnerabilities. + +## v0.6.1 [2025-07-02] +_Bug fixes_ +* Update core version to v0.2.9 - fix issue where collection state is not being saved for zero granularity collection. ([#251](https://github.com/turbot/tailpipe-plugin-sdk/issues/251)) + +## v0.6.0 [2025-07-02] +_What's new_ +* Add `--to` flag for `collect`, allowing collection of standalone time ranges. ([#238](https://github.com/turbot/tailpipe/issues/238)) +* Add `--overwrite` flag for `collect`, allowing recollection of existing data. ([#454](https://github.com/turbot/tailpipe/issues/454)) + +_Bug fixes_ +* Fix issue where collection state end-objects are cleared when collection is complete, +meaning no further data will be collected for that day. ([#250](https://github.com/turbot/tailpipe-plugin-sdk/issues/250)) + +_Behaviour Change_ + +When passing a `from` time to a collection, the existing partition data is no longer cleared before the collection starts. +This means that data will not by default be recollected for time ranges that have already been collected. +To recollect data for a time range, pass the new `--overwrite` flag to the `collect` command. + +## v0.5.0 [2025-06-20] +_What's new_ +* Added `tp_index` property to partition HCL. Use this to specify the source column for the `tp_index`. ([#414](https://github.com/turbot/tailpipe/issues/414)) +* Updated collection to apply the configured `tp_index`, or `default` if no `tp_index` is specified in the config. +* Added `--reindex` arg to `compact`. When set, compact will reindex the partition using configured `tp_index` value. ([#413](https://github.com/turbot/tailpipe/issues/413)) + - Removed the partition argument from compact and replaced it with a positional argument. + - Updated `compact` cleanup to delete empty folders. +* `collect` now always validates required columns are present. (Previously this was only done for custom tables.) ([#411](https://github.com/turbot/tailpipe/issues/411)) + +## v0.4.2 [2025-06-05] +_What's new_ +* Enabled support for collecting only today's logs during log collection. ([#394](https://github.com/turbot/tailpipe/issues/394)) +* Show available table names in autocomplete for DuckDB meta queries in interactive prompt. ([#357](https://github.com/turbot/tailpipe/issues/357)) +* `.inspect` meta-command now shows `tp_` columns at the end. ([#401](https://github.com/turbot/tailpipe/issues/401)) + +## v0.4.1 [2025-05-19] +_Bug fixes_ +* Update `MinCorePluginVersion` to v0.2.5. +* Fix issue where the core plugin was incorrectly throttling the downloads if no temp size limit was specified. ([#204](https://github.com/turbot/tailpipe-plugin-sdk/issues/204)) + +## v0.4.0 [2025-05-16] +_What's new_ +* Add support for memory and temp storage limits for CLI and plugins. ([#396](https://github.com/turbot/tailpipe/issues/396), [#397](https://github.com/turbot/tailpipe/issues/397)) + * `memory_max_mb` controls CLI memory usage and conversion worker count and memory allocation. + * `plugin_memory_max_mb` controls a per-plugin soft memory cap. + * `temp_dir_max_mb` limits size of temp data written to disk during a conversion. + * conversion worker count is now based on memory limit, if set. + * JSONL to Parquet conversion is now executed in multiple passes, limiting the number of distinct partition keys per conversion. +* Detect and report when a plugin crashes. ([#341](https://github.com/turbot/tailpipe/issues/341)) +* Update `show source` output to include source properties. ([#388](https://github.com/turbot/tailpipe/issues/388)) + +_Bug fixes_ +* Fix issue where tailpipe was mentioning steampipe in one of the error messages. ([#389](https://github.com/turbot/tailpipe/issues/389)) + +## v0.3.2 [2025-04-25] +_What's new_ +* Update `MinCorePluginVersion` to v0.2.2. +* Update tailpipe-plugin-sdk to v0.4.0. + +_Bug fixes_ +* Fix source file error for custom tables when using S3 or other external sources. ([#188](https://github.com/turbot/tailpipe-plugin-sdk/issues/188)) + +## v0.3.1 [2025-04-18] +_Bug fixes_ +* Fix partition filter argument. ([#375](https://github.com/turbot/tailpipe/issues/375)) + +## v0.3.0 [2025-04-16] + _What's new_ * Add support for custom tables. (([#225](https://github.com/turbot/tailpipe/issues/225))) * Add location to format list/show. ([#283](https://github.com/turbot/tailpipe/issues/283)) diff --git a/Makefile b/Makefile index be9846cb..74f25773 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ OUTPUT_DIR?=/usr/local/bin PACKAGE_NAME := github.com/turbot/tailpipe -GOLANG_CROSS_VERSION ?= v1.23.2 +GOLANG_CROSS_VERSION ?= gcc13-osxcross-20251006102018 # sed 's/[\/_]/-/g': Replaces both slashes (/) and underscores (_) with hyphens (-). # sed 's/[^a-zA-Z0-9.-]//g': Removes any character that isnā€™t alphanumeric, a dot (.), or a hyphen (-). @@ -23,13 +23,14 @@ release-dry-run: -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ -w /go/src/tailpipe \ - ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ + ghcr.io/turbot/goreleaser-cross:${GOLANG_CROSS_VERSION} \ --clean --skip=validate --skip=publish --snapshot .PHONY: release-acceptance release-acceptance: @docker run \ --rm \ + --platform=linux/arm64 \ -e CGO_ENABLED=1 \ -v /var/run/docker.sock:/var/run/docker.sock \ -v `pwd`:/go/src/tailpipe \ @@ -37,7 +38,7 @@ release-acceptance: -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ -w /go/src/tailpipe \ - ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ + ghcr.io/turbot/goreleaser-cross:${GOLANG_CROSS_VERSION} \ --clean --skip=validate --skip=publish --snapshot --config=.acceptance.goreleaser.yml .PHONY: release @@ -48,6 +49,7 @@ release: fi docker run \ --rm \ + --platform=linux/arm64 \ -e CGO_ENABLED=1 \ --env-file .release-env \ -v /var/run/docker.sock:/var/run/docker.sock \ @@ -56,5 +58,5 @@ release: -v `pwd`/../tailpipe-plugin-sdk:/go/src/tailpipe-plugin-sdk \ -v `pwd`/../tailpipe-plugin-core:/go/src/tailpipe-plugin-core \ -w /go/src/tailpipe \ - ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ - release --clean --skip=validate + ghcr.io/turbot/goreleaser-cross:${GOLANG_CROSS_VERSION} \ + release --clean --skip=validate \ No newline at end of file diff --git a/README.md b/README.md index 3daa5a14..e8ae6611 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -[![plugins](https://img.shields.io/badge/plugins-5-blue)](https://hub.tailpipe-io.vercel.app/)   -[![plugins](https://img.shields.io/badge/mods-14-blue)](https://hub.tailpipe-io.vercel.app/)   -[![slack](https://img.shields.io/badge/slack-2695-blue)](https://turbot.com/community/join?utm_id=gspreadme&utm_source=github&utm_medium=repo&utm_campaign=github&utm_content=readme)   +[![plugins](https://img.shields.io/endpoint?url=https://turbot.com/api/badge-stats?stat=tp_plugins)](https://hub.tailpipe.io/)   +[![mods](https://img.shields.io/endpoint?url=https://turbot.com/api/badge-stats?stat=tp_mods)](https://hub.tailpipe.io/)   +[![slack](https://img.shields.io/endpoint?url=https://turbot.com/api/badge-stats?stat=slack)](https://turbot.com/community/join?utm_id=gspreadme&utm_source=github&utm_medium=repo&utm_campaign=github&utm_content=readme)   [![maintained by](https://img.shields.io/badge/maintained%20by-Turbot-blue)](https://turbot.com?utm_id=gspreadme&utm_source=github&utm_medium=repo&utm_campaign=github&utm_content=readme) # select * from logs; diff --git a/cmd/collect.go b/cmd/collect.go index 52f163b7..80f228af 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -5,21 +5,25 @@ import ( "errors" "fmt" "log/slog" + "os" + "strconv" "strings" "time" - "github.com/danwakefield/fnmatch" + "github.com/hashicorp/hcl/v2" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" - "github.com/turbot/pipe-fittings/v2/error_helpers" + "github.com/turbot/pipe-fittings/v2/modconfig" "github.com/turbot/pipe-fittings/v2/parse" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/collector" "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/parquet" + "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/database" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" "github.com/turbot/tailpipe/internal/plugin" "golang.org/x/exp/maps" ) @@ -48,14 +52,17 @@ Every time you run tailpipe collect, Tailpipe refreshes its views over all colle cmdconfig.OnCmd(cmd). AddBoolFlag(pconstants.ArgCompact, true, "Compact the parquet files after collection"). AddStringFlag(pconstants.ArgFrom, "", "Collect days newer than a relative or absolute date (collection defaulting to 7 days if not specified)"). - AddBoolFlag(pconstants.ArgProgress, true, "Show active progress of collection, set to false to disable") + AddStringFlag(pconstants.ArgTo, "", "Collect days older than a relative or absolute date (defaulting to now if not specified)"). + AddBoolFlag(pconstants.ArgProgress, true, "Show active progress of collection, set to false to disable"). + AddBoolFlag(pconstants.ArgOverwrite, false, "Recollect data from the source even if it has already been collected") return cmd } func runCollectCmd(cmd *cobra.Command, args []string) { - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() + ctx, cancel := context.WithCancel(ctx) //nolint:govet // cancel is needed for the doCollect func var err error defer func() { @@ -64,30 +71,49 @@ func runCollectCmd(cmd *cobra.Command, args []string) { } if err != nil { - error_helpers.ShowError(ctx, err) + if error_helpers.IsCancelledError(err) { + fmt.Println("tailpipe collect command cancelled.") //nolint:forbidigo // ui output + } else { + error_helpers.ShowError(ctx, err) + } setExitCodeForCollectError(err) } }() - err = doCollect(ctx, cancel, args) - if errors.Is(err, context.Canceled) { - // clear error so we don't show it with normal error reporting - err = nil - fmt.Println("Collection cancelled.") //nolint:forbidigo // ui output + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return //nolint:govet // this is explicitly used in tests } + + err = doCollect(ctx, cancel, args) + } func doCollect(ctx context.Context, cancel context.CancelFunc, args []string) error { // arg `from` accepts ISO 8601 date(2024-01-01), ISO 8601 datetime(2006-01-02T15:04:05), ISO 8601 datetime with ms(2006-01-02T15:04:05.000), // RFC 3339 datetime with timezone(2006-01-02T15:04:05Z07:00) and relative time formats(T-2Y, T-10m, T-10W, T-180d, T-9H, T-10M) var fromTime time.Time + // toTime defaults to now, but can be set to a specific time + toTime := time.Now() + var err error if viper.IsSet(pconstants.ArgFrom) { - var err error - fromTime, err = parseFromTime(viper.GetString(pconstants.ArgFrom), time.Hour*24) + fromTime, err = parseFromToTime(viper.GetString(pconstants.ArgFrom)) + if err != nil { + return err + } + } + if viper.IsSet(pconstants.ArgTo) { + toTime, err = parseFromToTime(viper.GetString(pconstants.ArgTo)) if err != nil { return err } } + // validate from and to times + if err = validateCollectionTimeRange(fromTime, toTime); err != nil { + return err + } + partitions, err := getPartitions(args) if err != nil { return fmt.Errorf("failed to get partition config: %w", err) @@ -97,7 +123,14 @@ func doCollect(ctx context.Context, cancel context.CancelFunc, args []string) er for _, partition := range partitions { partitionNames = append(partitionNames, partition.FullName) } - slog.Info("Starting collection", "partition(s)", partitionNames, "from", fromTime) + slog.Info("Starting collection", "partition(s)", partitionNames, "from", fromTime, "to", toTime) + + // Create backup of metadata database before starting collection + if err := database.BackupDucklakeMetadata(); err != nil { + slog.Warn("Failed to backup metadata database", "error", err) + // Continue with collection - backup failure shouldn't block the operation + } + // now we have the partitions, we can start collecting // start the plugin manager @@ -106,18 +139,10 @@ func doCollect(ctx context.Context, cancel context.CancelFunc, args []string) er // collect each partition serially var errList []error + for _, partition := range partitions { - // if a from time is set, clear the partition data from that time forward - if !fromTime.IsZero() { - _, err := parquet.DeleteParquetFiles(partition, fromTime) - if err != nil { - slog.Warn("Failed to delete parquet files after the from time", "partition", partition.Name, "from", fromTime, "error", err) - errList = append(errList, err) - continue - } - } // do the collection - err = collectPartition(ctx, cancel, partition, fromTime, pluginManager) + err = collectPartition(ctx, cancel, partition, fromTime, toTime, pluginManager) if err != nil { errList = append(errList, err) } @@ -131,14 +156,27 @@ func doCollect(ctx context.Context, cancel context.CancelFunc, args []string) er return nil } -func collectPartition(ctx context.Context, cancel context.CancelFunc, partition *config.Partition, fromTime time.Time, pluginManager *plugin.PluginManager) (err error) { +func validateCollectionTimeRange(fromTime time.Time, toTime time.Time) error { + if !fromTime.IsZero() && !toTime.IsZero() && fromTime.After(toTime) { + return fmt.Errorf("invalid time range: 'from' time %s is after 'to' time %s", fromTime.Format(time.DateOnly), toTime.Format(time.DateOnly)) + } + if toTime.After(time.Now()) { + return fmt.Errorf("invalid time range: 'to' time %s is in the future", toTime.Format(time.DateOnly)) + } + return nil +} + +func collectPartition(ctx context.Context, cancel context.CancelFunc, partition *config.Partition, fromTime time.Time, toTime time.Time, pluginManager *plugin.PluginManager) (err error) { + t := time.Now() c, err := collector.New(pluginManager, partition, cancel) if err != nil { return fmt.Errorf("failed to create collector: %w", err) } defer c.Close() - if err = c.Collect(ctx, fromTime); err != nil { + overwrite := viper.GetBool(pconstants.ArgOverwrite) + + if err = c.Collect(ctx, fromTime, toTime, overwrite); err != nil { return err } @@ -149,13 +187,14 @@ func collectPartition(ctx context.Context, cancel context.CancelFunc, partition return err } - slog.Info("Collection complete", "partition", partition.Name) + slog.Info("Collection complete", "partition", partition.Name, "duration", time.Since(t).Seconds()) // compact the parquet files if viper.GetBool(pconstants.ArgCompact) { err = c.Compact(ctx) if err != nil { return err } + } // update status to show complete and display collection summary @@ -164,6 +203,7 @@ func collectPartition(ctx context.Context, cancel context.CancelFunc, partition return nil } +// getPartitions resolves the provided args to a list of partitions. func getPartitions(args []string) ([]*config.Partition, error) { // we have loaded tailpipe config by this time tailpipeConfig := config.GlobalConfig @@ -177,7 +217,12 @@ func getPartitions(args []string) ([]*config.Partition, error) { var partitions []*config.Partition for _, arg := range args { - partitionNames, err := getPartitionsForArg(maps.Keys(tailpipeConfig.Partitions), arg) + if syntheticPartition, ok := getSyntheticPartition(arg); ok { + partitions = append(partitions, syntheticPartition) + continue + } + + partitionNames, err := database.GetPartitionsForArg(tailpipeConfig.Partitions, arg) if err != nil { errorList = append(errorList, err) } else if len(partitionNames) == 0 { @@ -190,73 +235,135 @@ func getPartitions(args []string) ([]*config.Partition, error) { } if len(errorList) > 0 { - // TODO #errors better formating/error message https://github.com/turbot/tailpipe/issues/106 - return nil, errors.Join(errorList...) + // Return a well-formatted multi-error with a count and indented bullet list + return nil, formatErrorsWithCount(errorList) } return partitions, nil } -func getPartitionsForArg(partitions []string, arg string) ([]string, error) { - tablePattern, partitionPattern, err := getPartitionMatchPatternsForArg(partitions, arg) - if err != nil { - return nil, err +// formatErrorsWithCount returns an error summarizing a list of errors with a count and indented lines +func formatErrorsWithCount(errs []error) error { + if len(errs) == 0 { + return nil } - // now match the partition - var res []string - for _, partition := range partitions { - pattern := tablePattern + "." + partitionPattern - if fnmatch.Match(pattern, partition, fnmatch.FNM_CASEFOLD) { - res = append(res, partition) - } + if len(errs) == 1 { + return errs[0] } - return res, nil -} -func getPartitionMatchPatternsForArg(partitions []string, arg string) (string, string, error) { - var tablePattern, partitionPattern string - parts := strings.Split(arg, ".") - switch len(parts) { - case 1: - var err error - tablePattern, partitionPattern, err = getPartitionMatchPatternsForSinglePartName(partitions, arg) - if err != nil { - return "", "", err + var b strings.Builder + b.WriteString(fmt.Sprintf("%d errors:\n", len(errs))) + for i, e := range errs { + b.WriteString(fmt.Sprintf(" %s", e.Error())) + if i < len(errs)-1 { + b.WriteString("\n") } - case 2: - // use the args as provided - tablePattern = parts[0] - partitionPattern = parts[1] - default: - return "", "", fmt.Errorf("invalid partition name: %s", arg) - } - return tablePattern, partitionPattern, nil + } + return errors.New(b.String()) } -// getPartitionMatchPatternsForSinglePartName returns the table and partition patterns for a single part name -// e.g. if the arg is "aws*" -func getPartitionMatchPatternsForSinglePartName(partitions []string, arg string) (string, string, error) { - var tablePattern, partitionPattern string - // '*' is not valid for a single part arg - if arg == "*" { - return "", "", fmt.Errorf("invalid partition name: %s", arg) +// getSyntheticPartition parses a synthetic partition specification string and creates a test partition configuration. +// This function enables testing and performance benchmarking by generating dummy data instead of collecting from real sources. +// +// Synthetic partition format: synthetic_cols_rows_chunk_ms +// Example: "synthetic_50cols_2000000rows_10000chunk_100ms" +// - 50cols: Number of columns to generate in the synthetic table +// - 2000000rows: Total number of rows to generate +// - 10000chunk: Number of rows per chunk (affects memory usage and processing) +// - 100ms: Delivery interval between chunks (simulates real-time data collection) +// +// The function validates the format and numeric values, returning a properly configured Partition +// with SyntheticMetadata that will be used by the collector to generate test data. +// +// Returns: +// - *config.Partition: The configured synthetic partition if parsing succeeds +// - bool: true if the argument was a valid synthetic partition, false otherwise +func getSyntheticPartition(arg string) (*config.Partition, bool) { + // Check if this is a synthetic partition by looking for the "synthetic_" prefix + if !strings.HasPrefix(arg, "synthetic_") { + return nil, false } - // check whether there is table with this name - // partitions is a list of Unqualified names, i.e. . - for _, partition := range partitions { - table := strings.Split(partition, ".")[0] - // if the arg matches a table name, set table pattern to the arg and partition pattern to * - if fnmatch.Match(arg, table, fnmatch.FNM_CASEFOLD) { - tablePattern = arg - partitionPattern = "*" - return tablePattern, partitionPattern, nil - } + // Parse the synthetic partition parameters by splitting on underscores + // Expected format: synthetic_cols_rows_chunk_ms + parts := strings.Split(arg, "_") + if len(parts) != 5 { + // Invalid format - synthetic partitions must have exactly 5 parts + slog.Debug("Synthetic partition parsing failed: invalid format", "arg", arg, "parts", len(parts), "expected", 5) + return nil, false + } + + // Extract and parse the numeric values from each part + // Remove the suffix to get just the numeric value + colsStr := strings.TrimSuffix(parts[1], "cols") + rowsStr := strings.TrimSuffix(parts[2], "rows") + chunkStr := strings.TrimSuffix(parts[3], "chunk") + intervalStr := strings.TrimSuffix(parts[4], "ms") + + // Parse columns count - determines how many columns the synthetic table will have + cols, err := strconv.Atoi(colsStr) + if err != nil { + // Invalid columns value, not a synthetic partition + slog.Debug("Synthetic partition parsing failed: invalid columns value", "arg", arg, "colsStr", colsStr, "error", err) + return nil, false + } + + // Parse rows count - total number of rows to generate + rows, err := strconv.Atoi(rowsStr) + if err != nil { + // Invalid rows value, not a synthetic partition + slog.Debug("Synthetic partition parsing failed: invalid rows value", "arg", arg, "rowsStr", rowsStr, "error", err) + return nil, false + } + + // Parse chunk size - number of rows per chunk (affects memory usage and processing efficiency) + chunk, err := strconv.Atoi(chunkStr) + if err != nil { + // Invalid chunk value, not a synthetic partition + slog.Debug("Synthetic partition parsing failed: invalid chunk value", "arg", arg, "chunkStr", chunkStr, "error", err) + return nil, false + } + + // Parse delivery interval - milliseconds between chunk deliveries (simulates real-time data flow) + interval, err := strconv.Atoi(intervalStr) + if err != nil { + // Invalid interval value, not a synthetic partition + slog.Debug("Synthetic partition parsing failed: invalid interval value", "arg", arg, "intervalStr", intervalStr, "error", err) + return nil, false + } + + // Validate the parsed values - all must be positive integers + if cols <= 0 || rows <= 0 || chunk <= 0 || interval <= 0 { + // Invalid values, not a synthetic partition + slog.Debug("Synthetic partition parsing failed: invalid values", "arg", arg, "cols", cols, "rows", rows, "chunk", chunk, "interval", interval) + return nil, false + } + + // Create a synthetic partition with proper HCL block structure + // This mimics the structure that would be created from a real HCL configuration file + block := &hcl.Block{ + Type: "partition", + Labels: []string{"synthetic", arg}, + } + + // Create the partition configuration with synthetic metadata + partition := &config.Partition{ + HclResourceImpl: modconfig.NewHclResourceImpl(block, fmt.Sprintf("partition.synthetic.%s", arg)), + TableName: "synthetic", // All synthetic partitions use the "synthetic" table name + TpIndexColumn: "'default'", // Use a default index column for synthetic data + SyntheticMetadata: &config.SyntheticMetadata{ + Columns: cols, // Number of columns to generate + Rows: rows, // Total number of rows to generate + ChunkSize: chunk, // Rows per chunk + DeliveryIntervalMs: interval, // Milliseconds between chunk deliveries + }, } - // so there IS NOT a table with this name - set table pattern to * and user provided partition name - tablePattern = "*" - partitionPattern = arg - return tablePattern, partitionPattern, nil + + // Set the unqualified name for the partition (used in logging and identification) + partition.UnqualifiedName = fmt.Sprintf("%s.%s", partition.TableName, partition.ShortName) + + slog.Debug("Synthetic partition parsed successfully", "arg", arg, "columns", cols, "rows", rows, "chunkSize", chunk, "deliveryIntervalMs", interval) + return partition, true } func setExitCodeForCollectError(err error) { @@ -264,53 +371,26 @@ func setExitCodeForCollectError(err error) { if exitCode != 0 || err == nil { return } + // set exit code for cancellation + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return + } - // TODO #errors - assign exit codes https://github.com/turbot/tailpipe/issues/106 - exitCode = 1 + exitCode = pconstants.ExitCodeCollectionFailed } -// parse the from time, validating the granularity -// for example, if the from arg is T-4H and the granularity is 1 day, that is an error -func parseFromTime(fromArg string, granularity time.Duration) (time.Time, error) { +// parse the from time +func parseFromToTime(arg string) (time.Time, error) { now := time.Now() - fromTime, err := parse.ParseTime(fromArg, now) + // validate the granularity + granularity := time.Hour * 24 + + fromTime, err := parse.ParseTime(arg, now) if err != nil { - return time.Time{}, fmt.Errorf("failed to parse 'from' argument: %w", err) - } - // ensure the from time passed is more than the granularity away from now - // and truncate to the granularity - if time.Since(fromTime) < granularity { - return time.Time{}, fmt.Errorf("'from' time must be at least %s in the past", formatDuration(granularity)) + return time.Time{}, fmt.Errorf("failed to parse '%s' argument: %w", arg, err) } - return fromTime.Truncate(granularity), nil -} -// HumanizeDuration converts a time.Duration into a human-readable string -func formatDuration(d time.Duration) string { - if d.Hours() >= 24 { - days := int(d.Hours() / 24) - if days == 1 { - return "1 day" - } - return fmt.Sprintf("%d days", days) - } else if d.Hours() >= 1 { - hours := int(d.Hours()) - if hours == 1 { - return "1 hour" - } - return fmt.Sprintf("%d hours", hours) - } else if d.Minutes() >= 1 { - minutes := int(d.Minutes()) - if minutes == 1 { - return "1 minute" - } - return fmt.Sprintf("%d minutes", minutes) - } else { - seconds := int(d.Seconds()) - if seconds == 1 { - return "1 second" - } - return fmt.Sprintf("%d seconds", seconds) - } + return fromTime.Truncate(granularity), nil } diff --git a/cmd/collect_test.go b/cmd/collect_test.go index 725c27dc..73fc0c99 100644 --- a/cmd/collect_test.go +++ b/cmd/collect_test.go @@ -1,254 +1,140 @@ package cmd import ( - "reflect" "testing" + + "github.com/turbot/tailpipe/internal/config" ) -func Test_getPartition(t *testing.T) { - type args struct { - partitions []string - name string - } +func Test_getSyntheticPartition(t *testing.T) { tests := []struct { - name string - args args - want []string - wantErr bool + name string + arg string + wantPart *config.Partition + wantOk bool }{ { - name: "Invalid partition name", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "*", - }, - wantErr: true, - }, - { - name: "Full partition name, exists", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "aws_s3_cloudtrail_log.p1", - }, - want: []string{"aws_s3_cloudtrail_log.p1"}, - }, - { - name: "Full partition name, does not exist", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "aws_s3_cloudtrail_log.p3", + name: "Valid synthetic partition", + arg: "synthetic_50cols_2000000rows_10000chunk_100ms", + wantOk: true, + wantPart: &config.Partition{ + TableName: "synthetic", + SyntheticMetadata: &config.SyntheticMetadata{ + Columns: 50, + Rows: 2000000, + ChunkSize: 10000, + DeliveryIntervalMs: 100, + }, }, - want: nil, }, { - name: "Table name", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "aws_s3_cloudtrail_log", - }, - want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "Not a synthetic partition", + arg: "aws_cloudtrail_log.p1", + wantOk: false, }, { - name: "Table name (exists) with wildcard", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "aws_s3_cloudtrail_log.*", - }, - want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "Invalid synthetic partition format - too few parts", + arg: "synthetic_50cols_2000000rows_10000chunk", + wantOk: false, }, { - name: "Table name (exists) with ?", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "aws_s3_cloudtrail_log.p?", - }, - want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "Invalid synthetic partition format - too many parts", + arg: "synthetic_50cols_2000000rows_10000chunk_100ms_extra", + wantOk: false, }, { - name: "Table name (exists) with non matching partition wildacard", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "aws_s3_cloudtrail_log.d*?", - }, - want: nil, + name: "Invalid synthetic partition - non-numeric columns", + arg: "synthetic_abccols_2000000rows_10000chunk_100ms", + wantOk: false, }, { - name: "Table name (does not exist)) with wildcard", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - name: "foo.*", - }, - want: nil, + name: "Invalid synthetic partition - non-numeric rows", + arg: "synthetic_50cols_abcrows_10000chunk_100ms", + wantOk: false, }, { - name: "Partition short name, exists", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, - name: "p1", - }, - want: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, + name: "Invalid synthetic partition - non-numeric chunk", + arg: "synthetic_50cols_2000000rows_abcchunk_100ms", + wantOk: false, }, { - name: "Table wildcard, partition short name, exists", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, - name: "*.p1", - }, - want: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, + name: "Invalid synthetic partition - non-numeric interval", + arg: "synthetic_50cols_2000000rows_10000chunk_abcms", + wantOk: false, }, { - name: "Partition short name, does not exist", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, - name: "p3", - }, - want: nil, + name: "Invalid synthetic partition - zero values", + arg: "synthetic_0cols_2000000rows_10000chunk_100ms", + wantOk: false, }, { - name: "Table wildcard, partition short name, does not exist", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, - name: "*.p3", - }, - want: nil, + name: "Invalid synthetic partition - negative values", + arg: "synthetic_-50cols_2000000rows_10000chunk_100ms", + wantOk: false, }, { - name: "Table wildcard, no dot", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, - name: "aws*", - }, - want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, + name: "Invalid synthetic partition - zero interval", + arg: "synthetic_50cols_2000000rows_10000chunk_0ms", + wantOk: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := getPartitionsForArg(tt.args.partitions, tt.args.name) - if (err != nil) != tt.wantErr { - t.Errorf("getPartitions() error = %v, wantErr %v", err, tt.wantErr) + gotPart, gotOk := getSyntheticPartition(tt.arg) + if gotOk != tt.wantOk { + t.Errorf("getSyntheticPartition() gotOk = %v, want %v", gotOk, tt.wantOk) return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("getPartitions() got = %v, want %v", got, tt.want) + if gotOk { + if gotPart.TableName != tt.wantPart.TableName { + t.Errorf("getSyntheticPartition() TableName = %v, want %v", gotPart.TableName, tt.wantPart.TableName) + } + if gotPart.SyntheticMetadata == nil { + t.Errorf("getSyntheticPartition() SyntheticMetadata is nil") + return + } + if gotPart.SyntheticMetadata.Columns != tt.wantPart.SyntheticMetadata.Columns { + t.Errorf("getSyntheticPartition() Columns = %v, want %v", gotPart.SyntheticMetadata.Columns, tt.wantPart.SyntheticMetadata.Columns) + } + if gotPart.SyntheticMetadata.Rows != tt.wantPart.SyntheticMetadata.Rows { + t.Errorf("getSyntheticPartition() Rows = %v, want %v", gotPart.SyntheticMetadata.Rows, tt.wantPart.SyntheticMetadata.Rows) + } + if gotPart.SyntheticMetadata.ChunkSize != tt.wantPart.SyntheticMetadata.ChunkSize { + t.Errorf("getSyntheticPartition() ChunkSize = %v, want %v", gotPart.SyntheticMetadata.ChunkSize, tt.wantPart.SyntheticMetadata.ChunkSize) + } + if gotPart.SyntheticMetadata.DeliveryIntervalMs != tt.wantPart.SyntheticMetadata.DeliveryIntervalMs { + t.Errorf("getSyntheticPartition() DeliveryIntervalMs = %v, want %v", gotPart.SyntheticMetadata.DeliveryIntervalMs, tt.wantPart.SyntheticMetadata.DeliveryIntervalMs) + } } }) } } -func Test_getPartitionMatchPatternsForArg(t *testing.T) { - type args struct { - partitions []string - arg string - } - tests := []struct { - name string - args args - wantTablePattern string - wantPartPattern string - wantErr bool +func Test_getSyntheticPartition_Logging(t *testing.T) { + // Test that logging works for various failure scenarios + testCases := []struct { + name string + arg string }{ - { - name: "Valid table and partition pattern", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - arg: "aws_s3_cloudtrail_log.p1", - }, - wantTablePattern: "aws_s3_cloudtrail_log", - wantPartPattern: "p1", - }, - { - name: "Wildcard partition pattern", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1"}, - arg: "aws_s3_cloudtrail_log.*", - }, - wantTablePattern: "aws_s3_cloudtrail_log", - wantPartPattern: "*", - }, - { - name: "Wildcard in table and partition both", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1"}, - arg: "aws*.*", - }, - wantTablePattern: "aws*", - wantPartPattern: "*", - }, - { - name: "Wildcard table pattern", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, - arg: "*.p1", - }, - wantTablePattern: "*", - wantPartPattern: "p1", - }, - { - name: "Invalid partition name", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - arg: "*", - }, - wantErr: true, - }, - { - name: "Table exists without partition", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, - arg: "aws_s3_cloudtrail_log", - }, - wantTablePattern: "aws_s3_cloudtrail_log", - wantPartPattern: "*", - }, - { - name: "Partition only, multiple tables", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, - arg: "p1", - }, - wantTablePattern: "*", - wantPartPattern: "p1", - }, - { - name: "Invalid argument with multiple dots", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1"}, - arg: "aws.s3.cloudtrail", - }, - wantErr: true, - }, - { - name: "Non-existing table name", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1"}, - arg: "non_existing_table.p1", - }, - wantTablePattern: "non_existing_table", - wantPartPattern: "p1", - }, - { - name: "Partition name does not exist", - args: args{ - partitions: []string{"aws_s3_cloudtrail_log.p1"}, - arg: "p2", - }, - wantTablePattern: "*", - wantPartPattern: "p2", - }, + {"Invalid format", "synthetic_50cols_2000000rows_10000chunk"}, + {"Invalid columns", "synthetic_abccols_2000000rows_10000chunk_100ms"}, + {"Invalid rows", "synthetic_50cols_abcrows_10000chunk_100ms"}, + {"Invalid chunk", "synthetic_50cols_2000000rows_abcchunk_100ms"}, + {"Invalid interval", "synthetic_50cols_2000000rows_10000chunk_abcms"}, + {"Zero values", "synthetic_0cols_2000000rows_10000chunk_100ms"}, + {"Valid partition", "synthetic_50cols_2000000rows_10000chunk_100ms"}, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotTablePattern, gotPartPattern, err := getPartitionMatchPatternsForArg(tt.args.partitions, tt.args.arg) - if (err != nil) != tt.wantErr { - t.Errorf("getPartitionMatchPatternsForArg() error = %v, wantErr %v", err, tt.wantErr) - return - } - if gotTablePattern != tt.wantTablePattern { - t.Errorf("getPartitionMatchPatternsForArg() gotTablePattern = %v, want %v", gotTablePattern, tt.wantTablePattern) - } - if gotPartPattern != tt.wantPartPattern { - t.Errorf("getPartitionMatchPatternsForArg() gotPartPattern = %v, want %v", gotPartPattern, tt.wantPartPattern) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // This test ensures the function doesn't panic and handles logging gracefully + // The actual log output would be visible when running with debug level enabled + _, ok := getSyntheticPartition(tc.arg) + + // Just verify the function completes without error + // The logging is a side effect that we can't easily test without capturing log output + if tc.name == "Valid partition" && !ok { + t.Errorf("Expected valid partition to return true") } }) } diff --git a/cmd/compact.go b/cmd/compact.go index 263f121a..d95d8e4e 100644 --- a/cmd/compact.go +++ b/cmd/compact.go @@ -2,9 +2,7 @@ package cmd import ( "context" - "errors" "fmt" - "golang.org/x/exp/maps" "log/slog" "os" "time" @@ -15,30 +13,32 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" - "github.com/turbot/pipe-fittings/v2/error_helpers" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/parquet" + "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/database" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" + "golang.org/x/exp/maps" ) func compactCmd() *cobra.Command { cmd := &cobra.Command{ Use: "compact [table|table.partition] [flags]", - Args: cobra.ArbitraryArgs, + Args: cobra.MaximumNArgs(1), Run: runCompactCmd, Short: "Compact multiple parquet files per day to one per day", Long: `Compact multiple parquet files per day to one per day.`, } cmdconfig.OnCmd(cmd). - AddStringSliceFlag(pconstants.ArgPartition, nil, "Specify the partitions to compact. If not specified, all partitions will be compacted.") + AddBoolFlag(pconstants.ArgReindex, false, "Update the tp_index field to the currently configured value.") return cmd } -func runCompactCmd(cmd *cobra.Command, _ []string) { +func runCompactCmd(cmd *cobra.Command, args []string) { var err error - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() defer func() { if r := recover(); r != nil { @@ -46,68 +46,81 @@ func runCompactCmd(cmd *cobra.Command, _ []string) { } if err != nil { setExitCodeForCompactError(err) - error_helpers.ShowError(ctx, err) + + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui + fmt.Println("tailpipe compact command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + slog.Info("Compacting parquet files") - // if the partition flag is set, build a set of partition patterns, one per arg - var patterns []parquet.PartitionPattern - if viper.IsSet(pconstants.ArgPartition) { - availablePartitions := config.GlobalConfig.Partitions - partitionArgs := viper.GetStringSlice(pconstants.ArgPartition) - // Get table and partition patterns - patterns, err = getPartitionPatterns(partitionArgs, maps.Keys(availablePartitions)) + db, err := database.NewDuckDb(database.WithDuckLake()) + error_helpers.FailOnError(err) + defer db.Close() + + // verify that the provided args resolve to at least one partition + if _, err := getPartitions(args); err != nil { error_helpers.FailOnError(err) - slog.Info("Build partition patterns", "patterns", patterns) } - status, err := doCompaction(ctx, patterns...) - if errors.Is(err, context.Canceled) { - // clear error so we don't show it with normal error reporting - err = nil + // Get table and partition patterns + patterns, err := database.GetPartitionPatternsForArgs(maps.Keys(config.GlobalConfig.Partitions), args...) + error_helpers.FailOnErrorWithMessage(err, "failed to get partition patterns") + + // Create backup of metadata database before starting compaction + if err := database.BackupDucklakeMetadata(); err != nil { + slog.Warn("Failed to backup metadata database", "error", err) + // Continue with compaction - backup failure shouldn't block the operation } - if err == nil { - // print the final status - statusString := status.VerboseString() - if statusString == "" { - statusString = "No files to compact." - } - if ctx.Err() != nil { - // instead show the status as cancelled - statusString = "Compaction cancelled: " + statusString - } + // do the compaction + status, err := doCompaction(ctx, db, patterns) + // print the final status + statusString := status.VerboseString() + if err == nil { fmt.Println(statusString) //nolint:forbidigo // ui } // defer block will show the error } -func doCompaction(ctx context.Context, patterns ...parquet.PartitionPattern) (parquet.CompactionStatus, error) { +func doCompaction(ctx context.Context, db *database.DuckDb, patterns []*database.PartitionPattern) (*database.CompactionStatus, error) { s := spinner.New( spinner.CharSets[14], 100*time.Millisecond, spinner.WithHiddenCursor(true), spinner.WithWriter(os.Stdout), ) + // if the flag was provided, migrate the tp_index files + reindex := viper.GetBool(pconstants.ArgReindex) // start and stop spinner around the processing s.Start() defer s.Stop() s.Suffix = " compacting parquet files" - // define func to update the spinner suffix with the number of files compacted - var status parquet.CompactionStatus - updateTotals := func(counts parquet.CompactionStatus) { - status.Update(counts) - s.Suffix = fmt.Sprintf(" compacting parquet files (%d files -> %d files)", status.Source, status.Dest) + var status = database.NewCompactionStatus() + + updateTotals := func(updatedStatus database.CompactionStatus) { + status = &updatedStatus + if status.Message != "" { + s.Suffix = " compacting parquet files: " + status.Message + } } // do compaction - err := parquet.CompactDataFiles(ctx, updateTotals, patterns...) + err := database.CompactDataFiles(ctx, db, updateTotals, reindex, patterns...) return status, err } @@ -117,5 +130,11 @@ func setExitCodeForCompactError(err error) { if exitCode != 0 || err == nil { return } - exitCode = 1 + // set exit code for cancellation + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return + } + + exitCode = pconstants.ExitCodeCompactFailed } diff --git a/cmd/connect.go b/cmd/connect.go index 2dcafd46..7b83a59d 100644 --- a/cmd/connect.go +++ b/cmd/connect.go @@ -4,8 +4,6 @@ import ( "context" "encoding/json" "fmt" - "golang.org/x/exp/maps" - "io" "log" "os" "path/filepath" @@ -19,14 +17,14 @@ import ( "github.com/turbot/pipe-fittings/v2/cmdconfig" "github.com/turbot/pipe-fittings/v2/connection" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/error_helpers" pfilepaths "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/parse" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/filepaths" - "github.com/turbot/tailpipe/internal/parquet" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" + "golang.org/x/exp/maps" ) // variable used to assign the output mode flag @@ -37,8 +35,41 @@ func connectCmd() *cobra.Command { Use: "connect [flags]", Args: cobra.ArbitraryArgs, Run: runConnectCmd, - Short: "Return a connection string for a database, with a schema determined by the provided parameters", - Long: `Return a connection string for a database, with a schema determined by the provided parameters.`, + Short: "Return the path of SQL script to initialise DuckDB to use the tailpipe database", + Long: `Return the path of SQL script to initialise DuckDB to use the tailpipe database. + +The generated SQL script contains: +- DuckDB extension installations (sqlite, ducklake) +- Database attachment configuration +- View definitions with optional filters + +Examples: + # Basic usage - generate init script + tailpipe connect + + # Filter by time range + tailpipe connect --from "2024-01-01" --to "2024-01-31" + + # Filter by specific partitions + tailpipe connect --partition "aws_cloudtrail_log.recent" + + # Filter by indexes with wildcards + tailpipe connect --index "prod-*" --index "staging" + + # Combine multiple filters + tailpipe connect --from "T-7d" --partition "aws.*" --index "prod-*" + + # Output as JSON + tailpipe connect --output json + +Time formats supported: + - ISO 8601 date: 2024-01-01 + - ISO 8601 datetime: 2024-01-01T15:04:05 + - RFC 3339 with timezone: 2024-01-01T15:04:05Z + - Relative time: T-7d, T-2Y, T-10m, T-180d + +The generated script can be used with DuckDB: + duckdb -init /path/to/generated/script.sql`, } // args `from` and `to` accept: @@ -62,67 +93,153 @@ func connectCmd() *cobra.Command { func runConnectCmd(cmd *cobra.Command, _ []string) { var err error - var databaseFilePath string + var initFilePath string + // use the signal-aware/cancelable context created upstream in preRunHook ctx := cmd.Context() defer func() { if r := recover(); r != nil { err = helpers.ToError(r) } - setExitCodeForConnectError(err) - displayOutput(ctx, databaseFilePath, err) + if err != nil { + if error_helpers.IsCancelledError(err) { + fmt.Println("tailpipe connect command cancelled.") //nolint:forbidigo // ui output + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForConnectError(err) + } + displayOutput(ctx, initFilePath, err) }() - databaseFilePath, err = generateDbFile(ctx) + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + + initFilePath, err = generateInitFile(ctx) // we are done - the defer block will print either the filepath (if successful) or the error (if not) -} -func generateDbFile(ctx context.Context) (string, error) { - databaseFilePath := generateTempDBFilename(config.GlobalWorkspaceProfile.GetDataDir()) +} +func generateInitFile(ctx context.Context) (string, error) { // cleanup the old db files if not in use - err := cleanupOldDbFiles() + err := cleanupOldInitFiles() if err != nil { return "", err } - // first build the filters + // generate a filename to write the init sql to, inside the data dir + initFilePath := generateInitFilename(config.GlobalWorkspaceProfile.GetDataDir()) + + // get the sql to attach readonly to the database + commands := database.GetDucklakeInitCommands(true) + + // build the filters from the to, from and index args + // these will be used in the view definitions filters, err := getFilters() if err != nil { return "", fmt.Errorf("error building filters: %w", err) } - // if there are no filters, just copy the db file - if len(filters) == 0 { - err = copyDBFile(filepaths.TailpipeDbFilePath(), databaseFilePath) - return databaseFilePath, err + // create a temporary duckdb instance pass to get the view definitions + db, err := database.NewDuckDb(database.WithDuckLakeReadonly()) + if err != nil { + return "", fmt.Errorf("failed to create duckdb: %w", err) } + defer db.Close() - // Open a DuckDB connection (creates the file if it doesn't exist) - db, err := database.NewDuckDb(database.WithDbFile(databaseFilePath)) + // get the view creation SQL, with filters applied + viewCommands, err := database.GetCreateViewsSql(ctx, db, filters...) + if err != nil { + return "", err + } + commands = append(commands, viewCommands...) + // now build a string + var str strings.Builder + for _, cmd := range commands { + str.WriteString(fmt.Sprintf("-- %s\n%s;\n\n", cmd.Description, cmd.Command)) + } + // write out the init file + err = os.WriteFile(initFilePath, []byte(str.String()), 0644) //nolint:gosec // we want the init file to be readable if err != nil { - return "", fmt.Errorf("failed to open DuckDB connection: %w", err) + return "", fmt.Errorf("failed to write init file: %w", err) } - defer db.Close() + return initFilePath, err +} + +// cleanupOldInitFiles deletes old db init files (older than a day) +func cleanupOldInitFiles() error { + baseDir := pfilepaths.GetDataDir() + log.Printf("[INFO] Cleaning up old init files in %s\n", baseDir) + cutoffTime := time.Now().Add(-constants.InitFileMaxAge) // Files older than 1 day + + // The baseDir ("$TAILPIPE_INSTALL_DIR/data") is expected to have subdirectories for different workspace + // profiles(default, work etc). Each subdirectory may contain multiple .db files. + // Example structure: + // data/ + // ā”œā”€ā”€ default/ + // ā”‚ ā”œā”€ā”€ tailpipe_init_20250115182129.sql + // ā”‚ ā”œā”€ā”€ tailpipe_init_20250115193816.sql + // ā”‚ ā””ā”€ā”€ ... + // ā”œā”€ā”€ work/ + // ā”‚ ā”œā”€ā”€ tailpipe_init_20250115182129.sql + // ā”‚ ā”œā”€ā”€ tailpipe_init_20250115193816.sql + // ā”‚ ā””ā”€ā”€ ... + // So we traverse all these subdirectories for each workspace and process the relevant files. + err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("error accessing path %s: %v", path, err) + } + + // skip directories and non-`.sql` files + if info.IsDir() || !strings.HasSuffix(info.Name(), ".sql") { + return nil + } + + // only process `tailpipe_init_*.sql` files + if !strings.HasPrefix(info.Name(), "tailpipe_init_") { + return nil + } + + // check if the file is older than the cutoff time + if info.ModTime().After(cutoffTime) { + log.Printf("[DEBUG] Skipping deleting file %s(%s) as it is not older than %s\n", path, info.ModTime().String(), cutoffTime) + return nil + } + + err = os.Remove(path) + if err != nil { + log.Printf("[INFO] Failed to delete db file %s: %v", path, err) + } else { + log.Printf("[DEBUG] Cleaned up old unused db file: %s\n", path) + } + + return nil + }) + + if err != nil { + return err + } + return nil - err = database.AddTableViews(ctx, db, filters...) - return databaseFilePath, err } -func displayOutput(ctx context.Context, databaseFilePath string, err error) { +func displayOutput(ctx context.Context, initFilePath string, err error) { switch viper.GetString(pconstants.ArgOutput) { case pconstants.OutputFormatText: if err == nil { // output the filepath - fmt.Println(databaseFilePath) //nolint:forbidigo // ui output + fmt.Println(initFilePath) //nolint:forbidigo // ui output } else { error_helpers.ShowError(ctx, err) } case pconstants.OutputFormatJSON: res := connection.TailpipeConnectResponse{ - DatabaseFilepath: databaseFilePath, + InitScriptPath: initFilePath, } if err != nil { res.Error = err.Error() @@ -140,6 +257,8 @@ func displayOutput(ctx context.Context, databaseFilePath string, err error) { } } +// getFilters builds a set of SQL filters based on the provided command line args +// supported args are `from`, `to`, `partition` and `index` func getFilters() ([]string, error) { var result []string if viper.IsSet(pconstants.ArgFrom) { @@ -152,9 +271,8 @@ func getFilters() ([]string, error) { return nil, fmt.Errorf("invalid date format for 'from': %s", from) } // format as SQL timestamp - fromDate := t.Format(time.DateOnly) fromTimestamp := t.Format(time.DateTime) - result = append(result, fmt.Sprintf("tp_date >= date '%s' and tp_timestamp >= timestamp '%s'", fromDate, fromTimestamp)) + result = append(result, fmt.Sprintf("tp_timestamp >= timestamp '%s'", fromTimestamp)) } if viper.IsSet(pconstants.ArgTo) { to := viper.GetString(pconstants.ArgTo) @@ -166,9 +284,8 @@ func getFilters() ([]string, error) { return nil, fmt.Errorf("invalid date format for 'to': %s", to) } // format as SQL timestamp - toDate := t.Format(time.DateOnly) toTimestamp := t.Format(time.DateTime) - result = append(result, fmt.Sprintf("tp_date <= date '%s' and tp_timestamp <= timestamp '%s'", toDate, toTimestamp)) + result = append(result, fmt.Sprintf("tp_timestamp <= timestamp '%s'", toTimestamp)) } if viper.IsSet(pconstants.ArgPartition) { // we have loaded tailpipe config by this time @@ -193,115 +310,10 @@ func getFilters() ([]string, error) { return result, nil } -// generateTempDBFilename generates a temporary filename with a timestamp -func generateTempDBFilename(dataDir string) string { - timestamp := time.Now().Format("20060102150405") // e.g., 20241031103000 - return filepath.Join(dataDir, fmt.Sprintf("tailpipe_%s.db", timestamp)) -} - -func setExitCodeForConnectError(err error) { - // if exit code already set, leave as is - // NOTE: DO NOT set exit code if the output format is JSON - if exitCode != 0 || err == nil || viper.GetString(pconstants.ArgOutput) == pconstants.OutputFormatJSON { - return - } - - exitCode = 1 -} - -// copyDBFile copies the source database file to the destination -func copyDBFile(src, dst string) error { - sourceFile, err := os.Open(src) - if err != nil { - return err - } - defer sourceFile.Close() - - destFile, err := os.Create(dst) - if err != nil { - return err - } - defer destFile.Close() - - _, err = io.Copy(destFile, sourceFile) - return err -} - -// cleanupOldDbFiles deletes old db files(older than a day) that are not in use -func cleanupOldDbFiles() error { - baseDir := pfilepaths.GetDataDir() - log.Printf("[INFO] Cleaning up old db files in %s\n", baseDir) - cutoffTime := time.Now().Add(-constants.DbFileMaxAge) // Files older than 1 day - - // The baseDir ("$TAILPIPE_INSTALL_DIR/data") is expected to have subdirectories for different workspace - // profiles(default, work etc). Each subdirectory may contain multiple .db files. - // Example structure: - // data/ - // ā”œā”€ā”€ default/ - // ā”‚ ā”œā”€ā”€ tailpipe_20250115182129.db - // ā”‚ ā”œā”€ā”€ tailpipe_20250115193816.db - // ā”‚ ā”œā”€ā”€ tailpipe.db - // ā”‚ ā””ā”€ā”€ ... - // ā”œā”€ā”€ work/ - // ā”‚ ā”œā”€ā”€ tailpipe_20250115182129.db - // ā”‚ ā”œā”€ā”€ tailpipe_20250115193816.db - // ā”‚ ā”œā”€ā”€ tailpipe.db - // ā”‚ ā””ā”€ā”€ ... - // So we traverse all these subdirectories for each workspace and process the relevant files. - err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return fmt.Errorf("error accessing path %s: %v", path, err) - } - - // skip directories and non-`.db` files - if info.IsDir() || !strings.HasSuffix(info.Name(), ".db") { - return nil - } - - // skip `tailpipe.db` file - if info.Name() == "tailpipe.db" { - return nil - } - - // only process `tailpipe_*.db` files - if !strings.HasPrefix(info.Name(), "tailpipe_") { - return nil - } - - // check if the file is older than the cutoff time - if info.ModTime().After(cutoffTime) { - log.Printf("[DEBUG] Skipping deleting file %s(%s) as it is not older than %s\n", path, info.ModTime().String(), cutoffTime) - return nil - } - - // check for a lock on the file - db, err := database.NewDuckDb(database.WithDbFile(path)) - if err != nil { - log.Printf("[INFO] Skipping deletion of file %s due to error: %v\n", path, err) - return nil - } - defer db.Close() - - // if no lock, delete the file - err = os.Remove(path) - if err != nil { - log.Printf("[INFO] Failed to delete db file %s: %v", path, err) - } else { - log.Printf("[DEBUG] Cleaned up old unused db file: %s\n", path) - } - - return nil - }) - - if err != nil { - return err - } - return nil -} - +// getPartitionSqlFilters builds SQL filters for the provided partition args func getPartitionSqlFilters(partitionArgs []string, availablePartitions []string) (string, error) { - // Get table and partition patterns using getPartitionPatterns - patterns, err := getPartitionPatterns(partitionArgs, availablePartitions) + // Get table and partition patterns using GetPartitionPatternsForArgs + patterns, err := database.GetPartitionPatternsForArgs(availablePartitions, partitionArgs...) if err != nil { return "", fmt.Errorf("error processing partition args: %w", err) } @@ -357,6 +369,7 @@ func getPartitionSqlFilters(partitionArgs []string, availablePartitions []string return sqlFilters, nil } +// getIndexSqlFilters builds SQL filters for the provided index args func getIndexSqlFilters(indexArgs []string) (string, error) { // Return empty if no indexes provided if len(indexArgs) == 0 { @@ -385,30 +398,34 @@ func getIndexSqlFilters(indexArgs []string) (string, error) { return sqlFilter, nil } -// getPartitionPatterns returns the table and partition patterns for the given partition args -func getPartitionPatterns(partitionArgs []string, partitions []string) ([]parquet.PartitionPattern, error) { - var res []parquet.PartitionPattern - for _, arg := range partitionArgs { - tablePattern, partitionPattern, err := getPartitionMatchPatternsForArg(partitions, arg) - if err != nil { - return nil, fmt.Errorf("error processing partition arg '%s': %w", arg, err) - } - - res = append(res, parquet.PartitionPattern{Table: tablePattern, Partition: partitionPattern}) - } - - return res, nil -} - // convert partition patterns with '*' wildcards to SQL '%' wildcards -func replaceWildcards(patterns []parquet.PartitionPattern) []parquet.PartitionPattern { - updatedPatterns := make([]parquet.PartitionPattern, len(patterns)) +func replaceWildcards(patterns []*database.PartitionPattern) []*database.PartitionPattern { + updatedPatterns := make([]*database.PartitionPattern, len(patterns)) for i, p := range patterns { - updatedPatterns[i] = parquet.PartitionPattern{ + updatedPatterns[i] = &database.PartitionPattern{ Table: strings.ReplaceAll(p.Table, "*", "%"), Partition: strings.ReplaceAll(p.Partition, "*", "%")} } return updatedPatterns } + +func setExitCodeForConnectError(err error) { + // if exit code already set, leave as is + // NOTE: DO NOT set exit code if the output format is JSON + if exitCode != 0 || err == nil || viper.GetString(pconstants.ArgOutput) == pconstants.OutputFormatJSON { + return + } + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return + } + exitCode = pconstants.ExitCodeConnectFailed +} + +// generateInitFilename generates a temporary filename with a timestamp +func generateInitFilename(dataDir string) string { + timestamp := time.Now().Format("20060102150405") // e.g., 20241031103000 + return filepath.Join(dataDir, fmt.Sprintf("tailpipe_init_%s.sql", timestamp)) +} diff --git a/cmd/connect_test.go b/cmd/connect_test.go deleted file mode 100644 index 62fb47ec..00000000 --- a/cmd/connect_test.go +++ /dev/null @@ -1,194 +0,0 @@ -package cmd - -import ( - "testing" -) - -func Test_getPartitionSqlFilters(t *testing.T) { - tests := []struct { - name string - partitions []string - args []string - wantFilters string - wantErr bool - }{ - { - name: "Basic partition filters with wildcard", - partitions: []string{ - "aws_cloudtrail_log.p1", - "aws_cloudtrail_log.p2", - "github_audit_log.p1", - }, - args: []string{"aws_cloudtrail_log.*", "github_audit_log.p1"}, - wantFilters: "tp_table = 'aws_cloudtrail_log' OR " + - "(tp_table = 'github_audit_log' and tp_partition = 'p1')", - wantErr: false, - }, - { - name: "Wildcard in table and exact partition", - partitions: []string{ - "aws_cloudtrail_log.p1", - "sys_logs.p2", - }, - args: []string{"aws*.p1", "sys_logs.*"}, - wantFilters: "(tp_table like 'aws%' and tp_partition = 'p1') OR " + - "tp_table = 'sys_logs'", - wantErr: false, - }, - { - name: "Exact table and partition", - partitions: []string{ - "aws_cloudtrail_log.p1", - }, - args: []string{"aws_cloudtrail_log.p1"}, - wantFilters: "(tp_table = 'aws_cloudtrail_log' and tp_partition = 'p1')", - wantErr: false, - }, - { - name: "Partition with full wildcard", - partitions: []string{ - "aws_cloudtrail_log.p1", - }, - args: []string{"aws_cloudtrail_log.*"}, - wantFilters: "tp_table = 'aws_cloudtrail_log'", - wantErr: false, - }, - { - name: "Table with full wildcard", - partitions: []string{ - "aws_cloudtrail_log.p1", - }, - args: []string{"*.p1"}, - wantFilters: "tp_partition = 'p1'", - wantErr: false, - }, - { - name: "Both table and partition with full wildcards", - partitions: []string{ - "aws_cloudtrail_log.p1", - }, - args: []string{"*.*"}, - wantFilters: "", - wantErr: false, - }, - { - name: "Empty input", - partitions: []string{"aws_cloudtrail_log.p1"}, - args: []string{}, - wantFilters: "", - wantErr: false, - }, - { - name: "Multiple wildcards in table and partition", - partitions: []string{ - "aws_cloudtrail_log.p1", - "sys_logs.p2", - }, - args: []string{"aws*log.p*"}, - wantFilters: "(tp_table like 'aws%log' and tp_partition like 'p%')", - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotFilters, err := getPartitionSqlFilters(tt.args, tt.partitions) - if (err != nil) != tt.wantErr { - t.Errorf("getPartitionSqlFilters() name = %s error = %v, wantErr %v", tt.name, err, tt.wantErr) - return - } - if gotFilters != tt.wantFilters { - t.Errorf("getPartitionSqlFilters() name = %s got = %v, want %v", tt.name, gotFilters, tt.wantFilters) - } - }) - } -} - -func Test_getIndexSqlFilters(t *testing.T) { - tests := []struct { - name string - indexArgs []string - wantFilters string - wantErr bool - }{ - { - name: "Multiple indexes with wildcards and exact values", - indexArgs: []string{"1234*", "456789012345", "98*76"}, - wantFilters: "cast(tp_index as varchar) like '1234%' OR " + - "tp_index = '456789012345' OR " + - "cast(tp_index as varchar) like '98%76'", - wantErr: false, - }, - { - name: "Single index with wildcard", - indexArgs: []string{"12345678*"}, - wantFilters: "cast(tp_index as varchar) like '12345678%'", - wantErr: false, - }, - { - name: "No input provided", - indexArgs: []string{}, - wantFilters: "", - wantErr: false, - }, - { - name: "Fully wildcarded index", - indexArgs: []string{"*"}, - wantFilters: "", - wantErr: false, - }, - { - name: "Exact numeric index", - indexArgs: []string{"123456789012"}, - wantFilters: "tp_index = '123456789012'", - wantErr: false, - }, - { - name: "Mixed patterns", - indexArgs: []string{"12*", "3456789", "9*76"}, - wantFilters: "cast(tp_index as varchar) like '12%' OR " + - "tp_index = '3456789' OR " + - "cast(tp_index as varchar) like '9%76'", - wantErr: false, - }, - { - name: "Multiple exact values", - indexArgs: []string{"123456789012", "987654321098"}, - wantFilters: "tp_index = '123456789012' OR tp_index = '987654321098'", - wantErr: false, - }, - { - name: "Leading and trailing spaces in exact value", - indexArgs: []string{" 123456789012 "}, - wantFilters: "tp_index = ' 123456789012 '", // Spaces preserved - wantErr: false, - }, - { - name: "Combination of wildcards and exact values", - indexArgs: []string{"*456*", "1234", "98*76"}, - wantFilters: "cast(tp_index as varchar) like '%456%' OR " + - "tp_index = '1234' OR " + - "cast(tp_index as varchar) like '98%76'", - wantErr: false, - }, - { - name: "Empty string as index", - indexArgs: []string{""}, - wantFilters: "tp_index = ''", - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotFilters, err := getIndexSqlFilters(tt.indexArgs) - if (err != nil) != tt.wantErr { - t.Errorf("getIndexSqlFilters() error = %v, wantErr %v", err, tt.wantErr) - return - } - if gotFilters != tt.wantFilters { - t.Errorf("getIndexSqlFilters() got = %v, want %v", gotFilters, tt.wantFilters) - } - }) - } -} diff --git a/cmd/format.go b/cmd/format.go index 15165e23..94f8759d 100644 --- a/cmd/format.go +++ b/cmd/format.go @@ -1,8 +1,8 @@ package cmd import ( - "context" "fmt" + "os" "strings" "github.com/spf13/cobra" @@ -10,12 +10,12 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" ) // variable used to assign the output mode flag @@ -67,18 +67,32 @@ func formatListCmd() *cobra.Command { } func runFormatListCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() utils.LogTime("runFormatListCmd start") + var err error defer func() { utils.LogTime("runFormatListCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe format list command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForFormatError(err) } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resources, err := display.ListFormatResources(ctx) error_helpers.FailOnError(err) @@ -91,8 +105,8 @@ func runFormatListCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return } } @@ -116,15 +130,23 @@ func formatShowCmd() *cobra.Command { } func runFormatShowCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() utils.LogTime("runFormatShowCmd start") + var err error defer func() { utils.LogTime("runFormatShowCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe format show command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForFormatError(err) } }() @@ -141,7 +163,21 @@ func runFormatShowCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return + } +} + +func setExitCodeForFormatError(err error) { + // set exit code only if an error occurred and no exit code is already set + if exitCode != 0 || err == nil { + return + } + // set exit code for cancellation + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return } + // no dedicated format exit code exists yet; use generic nonzero failure + exitCode = 1 } diff --git a/cmd/partition.go b/cmd/partition.go index e7d6142f..7faf8be2 100644 --- a/cmd/partition.go +++ b/cmd/partition.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "errors" "fmt" "log/slog" "os" @@ -14,15 +15,16 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" + "github.com/turbot/pipe-fittings/v2/statushooks" "github.com/turbot/pipe-fittings/v2/utils" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/display" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" "github.com/turbot/tailpipe/internal/filepaths" - "github.com/turbot/tailpipe/internal/parquet" "github.com/turbot/tailpipe/internal/plugin" ) @@ -73,20 +75,38 @@ func partitionListCmd() *cobra.Command { } func runPartitionListCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() utils.LogTime("runPartitionListCmd start") + var err error defer func() { utils.LogTime("runPartitionListCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("taillpipe partition list command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPartitionError(err) } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + + db, err := database.NewDuckDb(database.WithDuckLakeReadonly()) + error_helpers.FailOnError(err) + defer db.Close() + // Get Resources - resources, err := display.ListPartitionResources(ctx) + resources, err := display.ListPartitionResources(ctx, db) error_helpers.FailOnError(err) printableResource := display.NewPrintableResource(resources...) @@ -97,15 +117,15 @@ func runPartitionListCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return } } // Show Partition func partitionShowCmd() *cobra.Command { var cmd = &cobra.Command{ - Use: "show", + Use: "show ", Args: cobra.ExactArgs(1), Run: runPartitionShowCmd, Short: "Show details for a specific partition", @@ -123,21 +143,53 @@ func partitionShowCmd() *cobra.Command { } func runPartitionShowCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + // TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a + // cancellation error. Cancellation won't work right now + ctx := cmd.Context() utils.LogTime("runPartitionShowCmd start") + var err error defer func() { utils.LogTime("runPartitionShowCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe partition show command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPartitionError(err) } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + + // open a readonly db connection + db, err := database.NewDuckDb(database.WithDuckLakeReadonly()) + error_helpers.FailOnError(err) + defer db.Close() + // Get Resources - partitionName := args[0] - resource, err := display.GetPartitionResource(partitionName) + + partitions, err := getPartitions(args) + error_helpers.FailOnError(err) + // if no partitions are found, return an error + if len(partitions) == 0 { + error_helpers.FailOnError(fmt.Errorf("no partitions found matching %s", args[0])) + } + // if more than one partition is found, return an error + if len(partitions) > 1 { + error_helpers.FailOnError(fmt.Errorf("multiple partitions found matching %s, please specify a more specific partition name", args[0])) + } + + resource, err := display.GetPartitionResource(cmd.Context(), partitions[0], db) error_helpers.FailOnError(err) printableResource := display.NewPrintableResource(resource) @@ -148,14 +200,14 @@ func runPartitionShowCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return } } func partitionDeleteCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "delete ", + Use: "delete ", Args: cobra.ExactArgs(1), Run: runPartitionDeleteCmd, Short: "Delete a partition for the specified period", @@ -171,32 +223,67 @@ func partitionDeleteCmd() *cobra.Command { cmdconfig.OnCmd(cmd). AddStringFlag(pconstants.ArgFrom, "", "Specify the start time"). + AddStringFlag(pconstants.ArgTo, "", "Specify the end time"). AddBoolFlag(pconstants.ArgForce, false, "Force delete without confirmation") return cmd } func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { + // use the signal-aware/cancelable context created upstream in preRunHook + // TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a + // cancellation error. Cancellation won't work right now ctx := cmd.Context() - + var err error defer func() { if r := recover(); r != nil { - exitCode = pconstants.ExitCodeUnknownErrorPanic - error_helpers.FailOnError(helpers.ToError(r)) + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("Partition cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPartitionError(err) } }() - // arg `fromTime` accepts ISO 8601 date(2024-01-01), ISO 8601 datetime(2006-01-02T15:04:05), ISO 8601 datetime with ms(2006-01-02T15:04:05.000), - // RFC 3339 datetime with timezone(2006-01-02T15:04:05Z07:00) and relative time formats(T-2Y, T-10m, T-10W, T-180d, T-9H, T-10M) + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // args `fromTime` and `ToTime` accepts: + // - ISO 8601 date(2024-01-01) + // - ISO 8601 datetime(2006-01-02T15:04:05) + // - ISO 8601 datetime with ms(2006-01-02T15:04:05.000) + // - RFC 3339 datetime with timezone(2006-01-02T15:04:05Z07:00) + // - relative time formats(T-2Y, T-10m, T-10W, T-180d, T-9H, T-10M) var fromTime time.Time - var fromStr string + // toTime defaults to now, but can be set to a specific time + toTime := time.Now() + // confirm deletion + var fromStr, toStr string + if viper.IsSet(pconstants.ArgFrom) { var err error - fromTime, err = parseFromTime(viper.GetString(pconstants.ArgFrom), time.Hour*24) - error_helpers.FailOnError(err) + fromTime, err = parseFromToTime(viper.GetString(pconstants.ArgFrom)) + error_helpers.FailOnErrorWithMessage(err, "invalid from time") fromStr = fmt.Sprintf(" from %s", fromTime.Format(time.DateOnly)) } + if viper.IsSet(pconstants.ArgTo) { + var err error + toTime, err = parseFromToTime(viper.GetString(pconstants.ArgTo)) + error_helpers.FailOnErrorWithMessage(err, "invalid to time") + } + toStr = fmt.Sprintf(" to %s", toTime.Format(time.DateOnly)) + if toTime.Before(fromTime) { + error_helpers.FailOnError(fmt.Errorf("to time %s cannot be before from time %s", toTime.Format(time.RFC3339), fromTime.Format(time.RFC3339))) + } + // retrieve the partition partitionName := args[0] partition, ok := config.GlobalConfig.Partitions[partitionName] if !ok { @@ -204,16 +291,37 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { } if !viper.GetBool(pconstants.ArgForce) { - // confirm deletion - msg := fmt.Sprintf("Are you sure you want to delete partition %s%s?", partitionName, fromStr) + msg := fmt.Sprintf("Are you sure you want to delete partition %s%s%s?", partitionName, fromStr, toStr) if !utils.UserConfirmationWithDefault(msg, true) { fmt.Println("Deletion cancelled") //nolint:forbidigo//expected output return } } - - filesDeleted, err := parquet.DeleteParquetFiles(partition, fromTime) + db, err := database.NewDuckDb(database.WithDuckLake()) error_helpers.FailOnError(err) + defer db.Close() + + // Create backup before deletion + slog.Info("Creating backup before partition deletion", "partition", partitionName) + if err := database.BackupDucklakeMetadata(); err != nil { + slog.Warn("Failed to create backup before partition deletion", "error", err) + // Continue with deletion - backup failure should not prevent deletion + } + + // show spinner while deleting the partition + spinner := statushooks.NewStatusSpinnerHook() + spinner.SetStatus(fmt.Sprintf("Deleting partition %s", partition.TableName)) + spinner.Show() + rowsDeleted, err := database.DeletePartition(ctx, partition, fromTime, toTime, db) + spinner.Hide() + if err != nil { + if errors.Is(err, context.Canceled) { + exitCode = pconstants.ExitCodeOperationCancelled + } else { + exitCode = 1 + } + error_helpers.FailOnError(err) + } // build the collection state path collectionStatePath := partition.CollectionStatePath(config.GlobalWorkspaceProfile.GetCollectionDir()) @@ -222,6 +330,7 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { if fromTime.IsZero() { err := os.Remove(collectionStatePath) if err != nil && !os.IsNotExist(err) { + exitCode = 1 error_helpers.FailOnError(fmt.Errorf("failed to delete collection state file: %s", err.Error())) } } else { @@ -230,24 +339,43 @@ func runPartitionDeleteCmd(cmd *cobra.Command, args []string) { pluginManager := plugin.NewPluginManager() defer pluginManager.Close() err = pluginManager.UpdateCollectionState(ctx, partition, fromTime, collectionStatePath) - error_helpers.FailOnError(err) + if err != nil { + if errors.Is(err, context.Canceled) { + exitCode = pconstants.ExitCodeOperationCancelled + } else { + exitCode = 1 + } + error_helpers.FailOnError(err) + } } // now prune the collection folders err = filepaths.PruneTree(config.GlobalWorkspaceProfile.GetCollectionDir()) if err != nil { - slog.Warn("DeleteParquetFiles failed to prune empty collection folders", "error", err) + slog.Warn("DeletePartition failed to prune empty collection folders", "error", err) } - msg := buildStatusMessage(filesDeleted, partitionName, fromStr) + msg := buildStatusMessage(rowsDeleted, partitionName, fromStr) fmt.Println(msg) //nolint:forbidigo//expected output } -func buildStatusMessage(filesDeleted int, partition string, fromStr string) interface{} { - var deletedStr = " (no parquet files deleted)" - if filesDeleted > 0 { - deletedStr = fmt.Sprintf(" (deleted %d parquet %s)", filesDeleted, utils.Pluralize("file", filesDeleted)) +func buildStatusMessage(rowsDeleted int, partition string, fromStr string) interface{} { + var deletedStr = " (nothing deleted)" + if rowsDeleted > 0 { + deletedStr = fmt.Sprintf(" (deleted %d %s)", rowsDeleted, utils.Pluralize("rows", rowsDeleted)) } return fmt.Sprintf("\nDeleted partition '%s'%s%s.\n", partition, fromStr, deletedStr) } + +func setExitCodeForPartitionError(err error) { + if exitCode != 0 || err == nil { + return + } + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return + } + // no dedicated partition exit code; use generic nonzero failure + exitCode = 1 +} diff --git a/cmd/plugin.go b/cmd/plugin.go index 55878629..25879a51 100644 --- a/cmd/plugin.go +++ b/cmd/plugin.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "os" "strings" "sync" "time" @@ -14,8 +15,7 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" - "github.com/turbot/pipe-fittings/v2/error_helpers" + "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/installationstate" pociinstaller "github.com/turbot/pipe-fittings/v2/ociinstaller" pplugin "github.com/turbot/pipe-fittings/v2/plugin" @@ -23,9 +23,11 @@ import ( "github.com/turbot/pipe-fittings/v2/statushooks" "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/pipe-fittings/v2/versionfile" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" "github.com/turbot/tailpipe/internal/ociinstaller" "github.com/turbot/tailpipe/internal/plugin" ) @@ -182,10 +184,9 @@ Examples: // Show plugin func pluginShowCmd() *cobra.Command { var cmd = &cobra.Command{ - Use: "show ", - Args: cobra.ExactArgs(1), - Run: runPluginShowCmd, - // TODO improve descriptions https://github.com/turbot/tailpipe/issues/111 + Use: "show ", + Args: cobra.ExactArgs(1), + Run: runPluginShowCmd, Short: "Show details of a plugin", Long: `Show the tables and sources provided by plugin`, } @@ -236,16 +237,37 @@ var pluginInstallSteps = []string{ } func runPluginInstallCmd(cmd *cobra.Command, args []string) { + // use the signal-aware/cancelable context created upstream in preRunHook + // TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a + // cancellation error. Cancellation won't work right now ctx := cmd.Context() utils.LogTime("runPluginInstallCmd install") + var err error defer func() { utils.LogTime("runPluginInstallCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe plugin install command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPluginError(err, 1) } }() + // Clean up plugin temporary directories from previous crashes/interrupted installations + filepaths.CleanupPluginTempDirs() + + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // args to 'plugin install' -- one or more plugins to install // plugin names can be simple names for "standard" plugins, constraint suffixed names // or full refs to the OCI image @@ -287,7 +309,7 @@ func runPluginInstallCmd(cmd *cobra.Command, args []string) { report := &pplugin.PluginInstallReport{ Plugin: pluginName, Skipped: true, - SkipReason: pconstants.InstallMessagePluginNotFound, + SkipReason: pconstants.InstallMessagePluginNotDistributedViaHub, IsUpdateReport: false, } reportChannel <- report @@ -363,16 +385,37 @@ func doPluginInstall(ctx context.Context, bar *uiprogress.Bar, pluginName string } func runPluginUpdateCmd(cmd *cobra.Command, args []string) { + // use the signal-aware/cancelable context created upstream in preRunHook + // TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a + // cancellation error. Cancellation won't work right now ctx := cmd.Context() utils.LogTime("runPluginUpdateCmd start") + var err error defer func() { utils.LogTime("runPluginUpdateCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe plugin update command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPluginError(err, 1) } }() + // Clean up plugin temporary directories from previous crashes/interrupted installations + filepaths.CleanupPluginTempDirs() + + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // args to 'plugin update' -- one or more plugins to update // These can be simple names for "standard" plugins, constraint suffixed names // or full refs to the OCI image @@ -441,6 +484,20 @@ func runPluginUpdateCmd(cmd *cobra.Command, args []string) { return } } else { + // Plugin not installed locally. If it's a hub plugin, check if it exists in hub. + org, name, constraint := ref.GetOrgNameAndStream() + if ref.IsFromTurbotHub() { + if _, err := pplugin.GetLatestPluginVersionByConstraint(ctx, state.InstallationID, org, name, constraint); err != nil { + updateResults = append(updateResults, &pplugin.PluginInstallReport{ + Skipped: true, + Plugin: p, + SkipReason: pconstants.InstallMessagePluginNotDistributedViaHub, + IsUpdateReport: true, + }) + continue + } + } + // Exists on hub (or not a hub plugin) but not installed locally exitCode = pconstants.ExitCodePluginNotFound updateResults = append(updateResults, &pplugin.PluginInstallReport{ Skipped: true, @@ -609,20 +666,46 @@ func installPlugin(ctx context.Context, resolvedPlugin pplugin.ResolvedPluginVer } func runPluginUninstallCmd(cmd *cobra.Command, args []string) { - // setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + // TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a + // cancellation error. Cancellation won't work right now + ctx := cmd.Context() utils.LogTime("runPluginUninstallCmd uninstall") - + var err error defer func() { utils.LogTime("runPluginUninstallCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe plugin uninstall command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPluginError(err, 1) } }() + // Clean up plugin temporary directories from previous crashes/interrupted installations + filepaths.CleanupPluginTempDirs() + + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + + // load installation state (needed for hub existence checks) + state, err := installationstate.Load() + if err != nil { + error_helpers.ShowError(ctx, fmt.Errorf("could not load state")) + exitCode = pconstants.ExitCodePluginLoadingError + return + } + if len(args) == 0 { fmt.Println() //nolint:forbidigo // ui output error_helpers.ShowError(ctx, fmt.Errorf("you need to provide at least one plugin to uninstall")) @@ -640,6 +723,18 @@ func runPluginUninstallCmd(cmd *cobra.Command, args []string) { if report, err := plugin.Remove(ctx, p); err != nil { if strings.Contains(err.Error(), "not found") { exitCode = pconstants.ExitCodePluginNotFound + // check hub existence to tailor message + ref := pociinstaller.NewImageRef(p) + if ref.IsFromTurbotHub() { + org, name, constraint := ref.GetOrgNameAndStream() + if _, herr := pplugin.GetLatestPluginVersionByConstraint(ctx, state.InstallationID, org, name, constraint); herr != nil { + // Not on hub and not installed locally + error_helpers.ShowError(ctx, fmt.Errorf("Failed to uninstall '%s' not found on hub and not installed locally.", p)) + continue + } + } + } else if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled } error_helpers.ShowErrorWithMessage(ctx, err, fmt.Sprintf("Failed to uninstall plugin '%s'", p)) } else { @@ -672,19 +767,37 @@ func resolveUpdatePluginsFromArgs(args []string) ([]string, error) { } func runPluginListCmd(cmd *cobra.Command, _ []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() utils.LogTime("runPluginListCmd list") + + // Clean up plugin temporary directories from previous crashes/interrupted installations + filepaths.CleanupPluginTempDirs() + + var err error defer func() { utils.LogTime("runPluginListCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe plugin list command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPluginError(err, pconstants.ExitCodePluginListFailure) } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resource(s) resources, err := display.ListPlugins(ctx) error_helpers.FailOnError(err) @@ -701,32 +814,52 @@ func runPluginListCmd(cmd *cobra.Command, _ []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodePluginListFailure + exitCode = pconstants.ExitCodeOutputRenderingFailed + return } } func runPluginShowCmd(cmd *cobra.Command, args []string) { + // use the signal-aware/cancelable context created upstream in preRunHook + // TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a + // cancellation error. Cancellation won't work right now + ctx := cmd.Context() + // we expect 1 argument, the plugin name if len(args) != 1 { - error_helpers.ShowError(cmd.Context(), fmt.Errorf("you need to provide the name of a plugin")) + error_helpers.ShowError(ctx, fmt.Errorf("you need to provide the name of a plugin")) exitCode = pconstants.ExitCodeInsufficientOrWrongInputs return } - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) - utils.LogTime("runPluginShowCmd start") + + // Clean up plugin temporary directories from previous crashes/interrupted installations + filepaths.CleanupPluginTempDirs() + + var err error defer func() { utils.LogTime("runPluginShowCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe plugin show command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForPluginError(err, pconstants.ExitCodePluginShowFailure) } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resource(s) resource, err := display.GetPluginResource(ctx, args[0]) error_helpers.FailOnError(err) @@ -739,7 +872,18 @@ func runPluginShowCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodePluginListFailure + exitCode = pconstants.ExitCodeOutputRenderingFailed + return + } +} + +func setExitCodeForPluginError(err error, nonCancelCode int) { + if exitCode != 0 || err == nil { + return + } + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return } + exitCode = nonCancelCode } diff --git a/cmd/query.go b/cmd/query.go index d767de7d..cdc6e011 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -1,8 +1,8 @@ package cmd import ( - "context" "fmt" + "os" "strings" "github.com/spf13/cobra" @@ -12,6 +12,7 @@ import ( "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/error_helpers" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/interactive" @@ -72,16 +73,27 @@ func runQueryCmd(cmd *cobra.Command, args []string) { } if err != nil { error_helpers.ShowError(ctx, err) - setExitCodeForQueryError(err) + exitCode = pconstants.ExitCodeInitializationFailed } }() - // get a connection to the database - var db *database.DuckDb - db, err = openDatabaseConnection(ctx) - if err != nil { + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() return } + + // build the filters from the to, from and index args + filters, err := getFilters() + if err != nil { + error_helpers.FailOnError(fmt.Errorf("error building filters: %w", err)) + } + + // now create a readonly connection to the database, passing in any filters + db, err := database.NewDuckDb(database.WithDuckLakeReadonly(filters...)) + if err != nil { + error_helpers.FailOnError(err) + } defer db.Close() // if an arg was passed, just execute the query @@ -99,25 +111,4 @@ func runQueryCmd(cmd *cobra.Command, args []string) { // if there were any errors, they would have been shown already from `RunBatchSession` - just set the exit code exitCode = pconstants.ExitCodeQueryExecutionFailed } - -} - -// generate a db file - this will respect any time/index filters specified in the command args -func openDatabaseConnection(ctx context.Context) (*database.DuckDb, error) { - dbFilePath, err := generateDbFile(ctx) - if err != nil { - return nil, err - } - // Open a DuckDB connection - return database.NewDuckDb(database.WithDbFile(dbFilePath)) -} - -func setExitCodeForQueryError(err error) { - // if exit code already set, leave as is - if exitCode != 0 || err == nil { - return - } - - // TODO #errors - assign exit codes https://github.com/turbot/tailpipe/issues/106 - exitCode = 1 } diff --git a/cmd/root.go b/cmd/root.go index d33ca2fa..402c3c0e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,6 +1,7 @@ package cmd import ( + "errors" "os" "github.com/spf13/cobra" @@ -10,8 +11,8 @@ import ( "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/utils" - localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/migration" ) var exitCode int @@ -35,7 +36,6 @@ func rootCommand() *cobra.Command { rootCmd.SetVersionTemplate("Tailpipe v{{.Version}}\n") - // TODO #config this will not reflect changes to install-dir - do we need to default in a different way https://github.com/turbot/tailpipe/issues/112 defaultConfigPath := filepaths.EnsureConfigDir() cmdconfig. @@ -63,18 +63,21 @@ func rootCommand() *cobra.Command { } func Execute() int { - // if diagnostic mode is set, print out config and return - if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { - localcmdconfig.DisplayConfig() - return 0 - } - - rootCmd := rootCommand() utils.LogTime("cmd.root.Execute start") defer utils.LogTime("cmd.root.Execute end") + rootCmd := rootCommand() + + // set the error output to stdout (as it;s common usage to redirect stderr to a file to capture logs + rootCmd.SetErr(os.Stdout) + // if the error is dues to unsupported migration, set a specific exit code - this will bve picked up by powerpipe if err := rootCmd.Execute(); err != nil { - exitCode = -1 + var unsupportedErr *migration.UnsupportedError + if errors.As(err, &unsupportedErr) { + exitCode = pconstants.ExitCodeMigrationUnsupported + } else { + exitCode = 1 + } } return exitCode } diff --git a/cmd/source.go b/cmd/source.go index 1067c36d..dcb9d5e6 100644 --- a/cmd/source.go +++ b/cmd/source.go @@ -1,8 +1,8 @@ package cmd import ( - "context" "fmt" + "os" "strings" "github.com/spf13/cobra" @@ -10,10 +10,10 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/display" ) @@ -64,18 +64,32 @@ func sourceListCmd() *cobra.Command { } func runSourceListCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() utils.LogTime("runSourceListCmd start") + var err error defer func() { utils.LogTime("runSourceListCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe source list command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForSourceError(err) } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resources, err := display.ListSourceResources(ctx) error_helpers.FailOnError(err) @@ -88,8 +102,8 @@ func runSourceListCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return } } @@ -113,18 +127,34 @@ func sourceShowCmd() *cobra.Command { } func runSourceShowCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + // TODO: https://github.com/turbot/tailpipe/issues/563 none of the functions called in this command will return a + // cancellation error. Cancellation won't work right now + ctx := cmd.Context() utils.LogTime("runSourceShowCmd start") + var err error defer func() { utils.LogTime("runSourceShowCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe source show command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForSourceError(err) } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + // Get Resources resourceName := args[0] resource, err := display.GetSourceResource(ctx, resourceName) @@ -138,7 +168,18 @@ func runSourceShowCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return + } +} + +func setExitCodeForSourceError(err error) { + if exitCode != 0 || err == nil { + return + } + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return } + exitCode = 1 } diff --git a/cmd/table.go b/cmd/table.go index 1ee88718..7574d84d 100644 --- a/cmd/table.go +++ b/cmd/table.go @@ -1,8 +1,8 @@ package cmd import ( - "context" "fmt" + "os" "strings" "github.com/spf13/cobra" @@ -10,12 +10,13 @@ import ( "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/contexthelpers" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/utils" + localcmdconfig "github.com/turbot/tailpipe/internal/cmdconfig" "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/display" + "github.com/turbot/tailpipe/internal/error_helpers" ) func tableCmd() *cobra.Command { @@ -65,20 +66,39 @@ func tableListCmd() *cobra.Command { } func runTableListCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() utils.LogTime("runSourceListCmd start") + var err error defer func() { utils.LogTime("runSourceListCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe table list command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForTableError(err) } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + + // open a readonly db connection + db, err := database.NewDuckDb(database.WithDuckLakeReadonly()) + error_helpers.FailOnError(err) + defer db.Close() + // Get Resources - resources, err := display.ListTableResources(ctx) + resources, err := display.ListTableResources(ctx, db) error_helpers.FailOnError(err) printableResource := display.NewPrintableResource(resources...) @@ -89,8 +109,8 @@ func runTableListCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return } } @@ -115,20 +135,39 @@ func tableShowCmd() *cobra.Command { } func runTableShowCmd(cmd *cobra.Command, args []string) { - //setup a cancel context and start cancel handler - ctx, cancel := context.WithCancel(cmd.Context()) - contexthelpers.StartCancelHandler(cancel) + // use the signal-aware/cancelable context created upstream in preRunHook + ctx := cmd.Context() utils.LogTime("runTableShowCmd start") + var err error defer func() { utils.LogTime("runTableShowCmd end") if r := recover(); r != nil { - error_helpers.ShowError(ctx, helpers.ToError(r)) - exitCode = pconstants.ExitCodeUnknownErrorPanic + err = helpers.ToError(r) + } + if err != nil { + if error_helpers.IsCancelledError(err) { + //nolint:forbidigo // ui output + fmt.Println("tailpipe table show command cancelled.") + } else { + error_helpers.ShowError(ctx, err) + } + setExitCodeForTableError(err) } }() + // if diagnostic mode is set, print out config and return + if _, ok := os.LookupEnv(constants.EnvConfigDump); ok { + localcmdconfig.DisplayConfig() + return + } + + // open a readonly db connection + db, err := database.NewDuckDb(database.WithDuckLakeReadonly()) + error_helpers.FailOnError(err) + defer db.Close() + // Get Resources - resource, err := display.GetTableResource(ctx, args[0]) + resource, err := display.GetTableResource(ctx, args[0], db) error_helpers.FailOnError(err) printableResource := display.NewPrintableResource(resource) @@ -139,7 +178,18 @@ func runTableShowCmd(cmd *cobra.Command, args []string) { // Print err = printer.PrintResource(ctx, printableResource, cmd.OutOrStdout()) if err != nil { - error_helpers.ShowError(ctx, err) - exitCode = pconstants.ExitCodeUnknownErrorPanic + exitCode = pconstants.ExitCodeOutputRenderingFailed + return + } +} + +func setExitCodeForTableError(err error) { + if exitCode != 0 || err == nil { + return + } + if error_helpers.IsCancelledError(err) { + exitCode = pconstants.ExitCodeOperationCancelled + return } + exitCode = 1 } diff --git a/gen/go.mod b/gen/go.mod deleted file mode 100644 index a7cc8e7a..00000000 --- a/gen/go.mod +++ /dev/null @@ -1,44 +0,0 @@ -module gen - -go 1.24.0 - -require ( - github.com/elastic/go-grok v0.3.1 - github.com/turbot/go-kit v1.0.0 - github.com/turbot/pipe-fittings/v2 v2.1.1 -) - -require ( - github.com/Masterminds/semver/v3 v3.2.1 // indirect - github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect - github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect - github.com/briandowns/spinner v1.23.0 // indirect - github.com/btubbs/datetime v0.1.1 // indirect - github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect - github.com/fatih/color v1.17.0 // indirect - github.com/gertd/go-pluralize v0.2.1 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect - github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/karrick/gows v0.3.0 // indirect - github.com/logrusorgru/aurora v2.0.3+incompatible // indirect - github.com/magefile/mage v1.15.0 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/rivo/uniseg v0.2.0 // indirect - github.com/shirou/gopsutil v3.21.11+incompatible // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/tklauser/go-sysconf v0.3.9 // indirect - github.com/tklauser/numcpus v0.3.0 // indirect - github.com/tkrajina/go-reflector v0.5.6 // indirect - github.com/yusufpapurcu/wmi v1.2.2 // indirect - github.com/zclconf/go-cty v1.14.4 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.27.0 // indirect - golang.org/x/text v0.21.0 // indirect -) diff --git a/gen/go.sum b/gen/go.sum deleted file mode 100644 index 99410adc..00000000 --- a/gen/go.sum +++ /dev/null @@ -1,90 +0,0 @@ -github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= -github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= -github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= -github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4Pt2A= -github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE= -github.com/btubbs/datetime v0.1.1 h1:KuV+F9tyq/hEnezmKZNGk8dzqMVsId6EpFVrQCfA3To= -github.com/btubbs/datetime v0.1.1/go.mod h1:n2BZ/2ltnRzNiz27aE3wUb2onNttQdC+WFxAoks5jJM= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= -github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/elastic/go-grok v0.3.1 h1:WEhUxe2KrwycMnlvMimJXvzRa7DoByJB4PVUIE1ZD/U= -github.com/elastic/go-grok v0.3.1/go.mod h1:n38ls8ZgOboZRgKcjMY8eFeZFMmcL9n2lP0iHhIDk64= -github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= -github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= -github.com/gertd/go-pluralize v0.2.1 h1:M3uASbVjMnTsPb0PNqg+E/24Vwigyo/tvyMTtAlLgiA= -github.com/gertd/go-pluralize v0.2.1/go.mod h1:rbYaKDbsXxmRfr8uygAEKhOWsjyrrqrkHVpZvoOp8zk= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/karrick/gows v0.3.0 h1:/FGSuBiJMUqNOJPsAdLvHFg7RnkFoWBS8USpdco5ONQ= -github.com/karrick/gows v0.3.0/go.mod h1:kdZ/jfdo8yqKYn+BMjBkhP+/oRKUABR1abaomzRi/n8= -github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= -github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= -github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= -github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= -github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= -github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= -github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= -github.com/tkrajina/go-reflector v0.5.6 h1:hKQ0gyocG7vgMD2M3dRlYN6WBBOmdoOzJ6njQSepKdE= -github.com/tkrajina/go-reflector v0.5.6/go.mod h1:ECbqLgccecY5kPmPmXg1MrHW585yMcDkVl6IvJe64T4= -github.com/turbot/go-kit v1.0.0 h1:6JlRbPdmvqT+5ca9fJldy1/zQr4Wwv05eZ8gIYktuSU= -github.com/turbot/go-kit v1.0.0/go.mod h1:vPk4gTUM8HhYGdzfKKLrPeZgnjLVBin41uqxjHScz6k= -github.com/turbot/pipe-fittings/v2 v2.1.1 h1:sV6bviX7WH3zivi45n29+ui+I9tJLlFNCNA2rOpw6/U= -github.com/turbot/pipe-fittings/v2 v2.1.1/go.mod h1:mGFH8dfDQOdv+d1fNL2r3ex+qlnVrTi3xGKZRVxoCEU= -github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= -github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8= -github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/gen/main.go b/gen/main.go deleted file mode 100644 index 86596ab8..00000000 --- a/gen/main.go +++ /dev/null @@ -1,132 +0,0 @@ -package main - -import ( - "fmt" - "os" - "path/filepath" - "strconv" - "time" - - "github.com/turbot/go-kit/files" - "github.com/turbot/pipe-fittings/v2/statushooks" -) - -const ( - ROWS_PER_FILE = 10000 // Number of rows per log file before creating a new file - FILES_PER_FOLDER = 10 // Number of files per folder before creating a new subfolder -) - -var ( - goodRowFormat = "%s 2 %d eni-0a86803e1af18f146 - - - - - - - 1720072205 1720072282 - NODATA\n" - errorRowFormat = "2 %d %s eni-0a86803e1af18f146 - - - - - - - 1720072205 1720072282 - NODATA\n" -) - -func main() { - if len(os.Args) != 4 { - fmt.Println("Usage: test_data ") - os.Exit(1) - } - - spinner := statushooks.NewStatusSpinnerHook(statushooks.WithMessage("Generating logs...")) - spinner.Show() - defer spinner.Hide() - totalRows, err := strconv.Atoi(os.Args[1]) - if err != nil || totalRows <= 0 { - fmt.Println("Invalid total rows value. Must be an positive integer.") - os.Exit(1) - } - - errorRows, err := strconv.Atoi(os.Args[2]) - if err != nil || errorRows < 0 { - fmt.Println("Invalid error rows value. Must be an integer.") - os.Exit(1) - } - - destDir, err := files.Tildefy(os.Args[3]) - if err != nil { - fmt.Println("Invalid destination directory:", err) - os.Exit(1) - } - // Create the destination directory, deleting if exists - if err := os.MkdirAll(destDir, 0755); err != nil { - fmt.Println("Error creating destination directory:", err) - os.Exit(1) - } - - // Calculate error interval - errorInterval, nextErrorRow := 0, 0 - if errorRows > 0 { - if errorRows >= totalRows { - // Every row will be an error row - errorInterval = 1 - } else { - errorInterval = totalRows / errorRows - } - nextErrorRow = errorInterval - } - - startTime := time.Now() - fileIndex := 1 - folderIndex := 1 - currentRow := 0 - var file *os.File - - for i := 1; i <= totalRows; i++ { - if i % 10000 == 0 { - spinner.SetStatus(fmt.Sprintf("Generating logs... %d/%d", i, totalRows)) - } - // Create a new folder if needed - if (fileIndex-1)%FILES_PER_FOLDER == 0 { - subFolder := filepath.Join(destDir, fmt.Sprintf("folder_%d", folderIndex)) - if err := os.MkdirAll(subFolder, 0755); err != nil { - fmt.Println("Error creating subdirectory:", err) - os.Exit(1) - } - folderIndex++ - } - - // Open a new file if needed - if currentRow%ROWS_PER_FILE == 0 { - if file != nil { - if err := file.Close(); err != nil { - fmt.Println("Error closing file:", err) - } - } - fileName := filepath.Join(destDir, fmt.Sprintf("folder_%d", folderIndex-1), fmt.Sprintf("logfile_%d.log", fileIndex)) - file, err = os.Create(fileName) - if err != nil { - fmt.Println("Error creating file:", err) - os.Exit(1) - } - fileIndex++ - } - - // Generate log entry - var logEntry string - if errorInterval > 0 && i == nextErrorRow { - logEntry = fmt.Sprintf(errorRowFormat, i, "ERROR") - nextErrorRow += errorInterval - } else { - timestamp := startTime.Add(time.Duration(i) * time.Millisecond).UTC().Format("2006-01-02T15:04:05.000Z") - logEntry = fmt.Sprintf(goodRowFormat, timestamp, i) - } - - // Write to file - if _, err := file.WriteString(logEntry); err != nil { - fmt.Println("Error writing to file:", err) - os.Exit(1) - } - - currentRow++ - } - - // Close the last file - if file != nil { - if err := file.Close(); err != nil { - fmt.Println("Error closing file:", err) - os.Exit(1) - } - } - - fmt.Println("Log generation complete. Files are in:", destDir) -} diff --git a/go.mod b/go.mod index f128b969..0a633771 100644 --- a/go.mod +++ b/go.mod @@ -8,22 +8,21 @@ replace ( github.com/c-bata/go-prompt => github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 //github.com/turbot/pipe-fittings/v2 => ../pipe-fittings //github.com/turbot/tailpipe-plugin-core => ../tailpipe-plugin-core -//github.com/turbot/tailpipe-plugin-sdk => ../tailpipe-plugin-sdk +// github.com/turbot/tailpipe-plugin-sdk => ../tailpipe-plugin-sdk ) require ( - github.com/Masterminds/semver/v3 v3.2.1 - github.com/hashicorp/hcl/v2 v2.20.1 + github.com/Masterminds/semver/v3 v3.4.0 + github.com/hashicorp/hcl/v2 v2.24.0 github.com/mattn/go-isatty v0.0.20 - github.com/spf13/cobra v1.8.1 + github.com/spf13/cobra v1.9.1 github.com/spf13/viper v1.19.0 - github.com/stretchr/testify v1.10.0 - github.com/turbot/go-kit v1.2.0 - github.com/turbot/pipe-fittings/v2 v2.3.3 - github.com/turbot/tailpipe-plugin-sdk v0.3.1 - github.com/zclconf/go-cty v1.14.4 - golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c - + github.com/stretchr/testify v1.11.0 + github.com/turbot/go-kit v1.3.0 + github.com/turbot/pipe-fittings/v2 v2.7.0 + github.com/turbot/tailpipe-plugin-sdk v0.9.3 + github.com/zclconf/go-cty v1.16.3 + golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 ) require ( @@ -33,39 +32,46 @@ require ( github.com/charmbracelet/bubbletea v1.2.4 github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 github.com/dustin/go-humanize v1.0.1 - github.com/fsnotify/fsnotify v1.8.0 + github.com/fatih/color v1.18.0 + github.com/fsnotify/fsnotify v1.9.0 github.com/gosuri/uiprogress v0.0.1 github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-plugin v1.6.1 github.com/hashicorp/go-version v1.7.0 github.com/jedib0t/go-pretty/v6 v6.5.9 - github.com/marcboeker/go-duckdb/v2 v2.1.0 + github.com/marcboeker/go-duckdb/v2 v2.4.0 + github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02 github.com/thediveo/enumflag/v2 v2.0.5 - github.com/turbot/tailpipe-plugin-core v0.2.1 - golang.org/x/sync v0.11.0 - golang.org/x/text v0.22.0 - google.golang.org/protobuf v1.36.1 + github.com/turbot/tailpipe-plugin-core v0.2.10 + golang.org/x/text v0.28.0 + google.golang.org/grpc v1.75.0 + google.golang.org/protobuf v1.36.8 ) require ( github.com/goccy/go-json v0.10.5 // indirect - github.com/google/flatbuffers v25.1.24+incompatible // indirect - github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/google/flatbuffers v25.2.10+incompatible // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect ) require ( - cloud.google.com/go v0.115.0 // indirect - cloud.google.com/go/auth v0.7.2 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect - cloud.google.com/go/compute/metadata v0.5.2 // indirect - cloud.google.com/go/iam v1.1.10 // indirect - cloud.google.com/go/storage v1.42.0 // indirect + cel.dev/expr v0.24.0 // indirect + cloud.google.com/go v0.121.0 // indirect + cloud.google.com/go/auth v0.16.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.7.0 // indirect + cloud.google.com/go/iam v1.5.0 // indirect + cloud.google.com/go/monitoring v1.24.0 // indirect + cloud.google.com/go/storage v1.52.0 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/agext/levenshtein v1.2.3 // indirect - github.com/apache/arrow-go/v18 v18.1.0 // indirect + github.com/apache/arrow-go/v18 v18.4.1 // indirect github.com/apparentlymart/go-cidr v1.1.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/aws/aws-sdk-go v1.44.183 // indirect @@ -91,6 +97,7 @@ require ( github.com/charmbracelet/lipgloss v1.0.0 // indirect github.com/charmbracelet/x/ansi v0.4.5 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect github.com/containerd/containerd v1.7.27 // indirect github.com/containerd/errdefs v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -99,40 +106,42 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/ristretto v0.2.0 // indirect github.com/dlclark/regexp2 v1.4.0 // indirect - github.com/duckdb/duckdb-go-bindings v0.1.13 // indirect - github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.8 // indirect - github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.8 // indirect - github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.8 // indirect - github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.8 // indirect - github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.8 // indirect + github.com/duckdb/duckdb-go-bindings v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.19 // indirect + github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.19 // indirect github.com/elastic/go-grok v0.3.1 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect - github.com/fatih/color v1.17.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gertd/go-pluralize v0.2.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.6.0 // indirect github.com/go-git/go-git/v5 v5.13.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-jose/go-jose/v4 v4.1.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect - github.com/go-viper/mapstructure/v2 v2.2.1 // indirect - github.com/goccy/go-yaml v1.11.2 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/goccy/go-yaml v1.17.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/s2a-go v0.1.7 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.13.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/gax-go/v2 v2.14.1 // indirect github.com/gosuri/uilive v0.0.4 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-getter v1.7.5 // indirect + github.com/hashicorp/go-getter v1.7.9 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect @@ -154,14 +163,14 @@ require ( github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/karrick/gows v0.3.0 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magefile/mage v1.15.0 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/marcboeker/go-duckdb/arrowmapping v0.0.6 // indirect - github.com/marcboeker/go-duckdb/mapping v0.0.6 // indirect + github.com/marcboeker/go-duckdb/arrowmapping v0.0.19 // indirect + github.com/marcboeker/go-duckdb/mapping v0.0.19 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -169,7 +178,7 @@ require ( github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect - github.com/mitchellh/go-wordwrap v1.0.0 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/locker v1.0.1 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect @@ -183,52 +192,58 @@ require ( github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/term v1.1.0 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/satyrius/gonx v1.4.0 // indirect - github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect github.com/stevenle/topsort v0.2.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tklauser/go-sysconf v0.3.9 // indirect - github.com/tklauser/numcpus v0.3.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/tkrajina/go-reflector v0.5.8 // indirect github.com/turbot/pipes-sdk-go v0.12.0 // indirect github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 // indirect - github.com/ulikunitz/xz v0.5.10 // indirect + github.com/ulikunitz/xz v0.5.14 // indirect github.com/xlab/treeprint v1.2.0 // indirect - github.com/yusufpapurcu/wmi v1.2.2 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zclconf/go-cty-yaml v1.0.3 // indirect - go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect - go.opentelemetry.io/otel v1.31.0 // indirect - go.opentelemetry.io/otel/metric v1.31.0 // indirect - go.opentelemetry.io/otel/trace v1.31.0 // indirect + github.com/zeebo/errs v1.4.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/sdk v1.37.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.35.0 // indirect - golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.36.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/term v0.29.0 // indirect - golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.29.0 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/mod v0.27.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/term v0.34.0 // indirect + golang.org/x/time v0.11.0 // indirect + golang.org/x/tools v0.36.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/api v0.189.0 // indirect - google.golang.org/genproto v0.0.0-20240722135656-d784300faade // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect - google.golang.org/grpc v1.69.2 // indirect + google.golang.org/api v0.230.0 // indirect + google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 68b19c72..1640c3f0 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,11 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= @@ -15,6 +18,7 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= @@ -26,32 +30,96 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= -cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= -cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= +cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.121.0 h1:pgfwva8nGw7vivjZiRfrmglGWiCJBP+0OmDpenG/Fwg= +cloud.google.com/go v0.121.0/go.mod h1:rS7Kytwheu/y9buoDmu5EIpMMCI4Mb8ND4aeN4Vwj7Q= +cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= +cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= +cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= +cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= +cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= +cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM= +cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= +cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ= +cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= +cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= +cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= +cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= +cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= +cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= +cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= +cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= +cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= +cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= +cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY= +cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM= +cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= +cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU= +cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI= +cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= +cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= +cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= +cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84= +cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A= +cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY= +cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= +cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= +cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI= +cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ= +cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI= +cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= +cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= +cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo= +cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg= +cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= -cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE= -cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= -cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= -cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= +cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= +cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= +cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= +cloud.google.com/go/auth v0.16.0 h1:Pd8P1s9WkcrBE2n/PhAwKsdrR35V3Sg2II9B+ndM3CU= +cloud.google.com/go/auth v0.16.0/go.mod h1:1howDHJ5IETh/LwYs3ZxvlkXF48aSqqJUM+5o02dNOI= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= +cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= +cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= +cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU= +cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= +cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= +cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= +cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= +cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= +cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= +cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= +cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= +cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= +cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -59,12 +127,44 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= +cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E= +cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= +cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= +cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= +cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= +cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= +cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= +cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= +cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= +cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= +cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= +cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= +cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= +cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= +cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= +cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= +cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= +cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= +cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= +cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= +cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= +cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= +cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= +cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= +cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= +cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= @@ -72,129 +172,466 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= -cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= +cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= +cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= +cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= +cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= +cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= +cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= +cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= +cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= +cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= +cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= +cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= +cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA= +cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= +cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= +cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= +cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM= +cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= +cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= +cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= +cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= +cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= +cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= +cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= +cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= +cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= +cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= +cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= +cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= +cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= +cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= +cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= +cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= +cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM= +cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= +cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= +cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= +cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= +cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= +cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= +cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= +cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc= +cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= +cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= +cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= +cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= +cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M= +cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= +cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= +cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= +cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= +cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= +cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= +cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= +cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= +cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= +cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw= +cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA= +cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= +cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= +cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= +cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= +cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= +cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E= +cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= +cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= +cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= +cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= +cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= +cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/iam v1.1.10 h1:ZSAr64oEhQSClwBL670MsJAW5/RLiC6kfw3Bqmd5ZDI= -cloud.google.com/go/iam v1.1.10/go.mod h1:iEgMq62sg8zx446GCaijmA2Miwg5o3UbO+nI47WHJps= +cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= +cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iam v1.5.0 h1:QlLcVMhbLGOjRcGe6VTGGTyQib8dRLK2B/kYNV0+2xs= +cloud.google.com/go/iam v1.5.0/go.mod h1:U+DOtKQltF/LxPEtcDLoobcsZMilSRwR7mgNL7knOpo= +cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= +cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= +cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= +cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= +cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= +cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= +cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= +cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= +cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= +cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= +cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o= +cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= +cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= +cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= +cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= +cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg= +cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= +cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= +cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= +cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= +cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= -cloud.google.com/go/longrunning v0.5.9 h1:haH9pAuXdPAMqHvzX0zlWQigXT7B0+CL4/2nXXdBo5k= -cloud.google.com/go/longrunning v0.5.9/go.mod h1:HD+0l9/OOW0za6UWdKJtXoFAX/BGg/3Wj8p10NeWF7c= +cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= +cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= +cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= +cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/longrunning v0.6.6 h1:XJNDo5MUfMM05xK3ewpbSdmt7R2Zw+aQEMbdQR65Rbw= +cloud.google.com/go/longrunning v0.6.6/go.mod h1:hyeGJUrPHcx0u2Uu1UFSoYZLn4lkMrccJig0t4FI7yw= +cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= +cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= +cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= +cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= +cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= +cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= +cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= +cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM= cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= +cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= +cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= +cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= +cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= +cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= +cloud.google.com/go/monitoring v1.24.0 h1:csSKiCJ+WVRgNkRzzz3BPoGjFhjPY23ZTcaenToJxMM= +cloud.google.com/go/monitoring v1.24.0/go.mod h1:Bd1PRK5bmQBQNnuGwHBfUamAV1ys9049oEPHnn4pcsc= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= +cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= +cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= +cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= +cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= +cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= +cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= +cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= +cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= +cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= +cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= +cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= +cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= +cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= +cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= +cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= +cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= +cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= +cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= +cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc= cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= +cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= +cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw= cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= +cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= +cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs= cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk= +cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= +cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= +cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= +cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= +cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= +cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= +cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= +cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= +cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= +cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= +cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= +cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= +cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= +cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= +cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= +cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= +cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ= +cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= +cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= +cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots= +cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo= +cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= +cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= +cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= +cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= +cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= +cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14= +cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= +cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= +cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= +cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= +cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= +cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc= +cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= +cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= +cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= +cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= +cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8= +cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= +cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= +cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0= +cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= +cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= +cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= +cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA= +cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc= +cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= +cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= +cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= +cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= +cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= +cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= +cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= +cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= +cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= +cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= +cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= +cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= +cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= +cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= +cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= +cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= +cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= +cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= +cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= +cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= +cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= +cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= -cloud.google.com/go/storage v1.42.0 h1:4QtGpplCVt1wz6g5o1ifXd656P5z+yNgzdw1tVfp0cU= -cloud.google.com/go/storage v1.42.0/go.mod h1:HjMXRFq65pGKFn6hxj6x3HCyR41uSB72Z0SO/Vn6JFQ= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storage v1.52.0 h1:ROpzMW/IwipKtatA69ikxibdzQSiXJrY9f6IgBa9AlA= +cloud.google.com/go/storage v1.52.0/go.mod h1:4wrBAbAYUvYkbrf19ahGm4I5kDQhESSqN3CGEkMGvOY= +cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= +cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= +cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= +cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= +cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= +cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c= +cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= +cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= +cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc= +cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= +cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= +cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM= +cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= +cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= +cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= +cloud.google.com/go/trace v1.11.3 h1:c+I4YFjxRQjvAhRmSsmjpASUKq88chOX854ied0K/pE= +cloud.google.com/go/trace v1.11.3/go.mod h1:pt7zCYiDSQjC9Y2oqCsh9jF4GStB/hmjrYLsxRR27q8= +cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= +cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= +cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= +cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= +cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= +cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= +cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk= +cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= +cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= +cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU= cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= +cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= +cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY= +cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= +cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= +cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= +cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc= +cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= +cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= +cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= +cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= +cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= +cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= +cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= +cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= +cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= +cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= +cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= +cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= +cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= +cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= +cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= -github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.11.7 h1:vl/nj3Bar/CvJSYo7gIQPyRWc9f3c6IeSNavBTSZNZQ= @@ -204,15 +641,23 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= +github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= -github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= -github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/arrow-go/v18 v18.1.0 h1:agLwJUiVuwXZdwPYVrlITfx7bndULJ/dggbnLFgDp/Y= -github.com/apache/arrow-go/v18 v18.1.0/go.mod h1:tigU/sIgKNXaesf5d7Y95jBBKS5KsxTqYBKXFsvKzo0= -github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= -github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= +github.com/apache/arrow-go/v18 v18.4.1 h1:q/jVkBWCJOB9reDgaIZIdruLQUb1kbkvOnOFezVH1C4= +github.com/apache/arrow-go/v18 v18.4.1/go.mod h1:tLyFubsAl17bvFdUAy24bsSvA/6ww95Iqi67fTpGu3E= +github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= +github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= github.com/apparentlymart/go-dump v0.0.0-20190214190832-042adf3cf4a0 h1:MzVXffFUye+ZcSR6opIgz9Co7WcDx6ZcY+RjfFHoA0I= @@ -256,6 +701,8 @@ github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4Pt2A= github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE= github.com/btubbs/datetime v0.1.1 h1:KuV+F9tyq/hEnezmKZNGk8dzqMVsId6EpFVrQCfA3To= @@ -263,8 +710,11 @@ github.com/btubbs/datetime v0.1.1/go.mod h1:n2BZ/2ltnRzNiz27aE3wUb2onNttQdC+WFxA github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbletea v1.2.4 h1:KN8aCViA0eps9SCOThb2/XPIlea3ANJLUkv3KnQRNCE= @@ -284,11 +734,17 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/containerd v1.7.27 h1:yFyEyojddO3MIGVER2xJLWoCIn+Up4GaHFquP7hsFII= @@ -301,7 +757,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.5 h1:6iR5tXJ/e6tJZzzdMc1km3Sa7RRIVBKAK32O2s7AYfo= github.com/cyphar/filepath-securejoin v0.2.5/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= @@ -316,18 +773,20 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WA github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/duckdb/duckdb-go-bindings v0.1.13 h1:3Ec0SjMBuzt7wExde5ZoMXd1Nk91LJmpopq2Ee6g9Pw= -github.com/duckdb/duckdb-go-bindings v0.1.13/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc= -github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.8 h1:n4RNMqiUPao53YKmlh36zGEr49CnUXGVKOtOMCEhwFE= -github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.8/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA= -github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.8 h1:3ZBS6wETlZp9UDmaWJ4O4k7ZSjqQjyhMW5aZZBXThqM= -github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.8/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI= -github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.8 h1:KCUI9KSAUKbYasNlTcjky30nbDtF18S6s6R3usXWLqk= -github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.8/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk= -github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.8 h1:QgKzpNG7EMPq3ayYcr0LzGfC+dCzGA/Gm6Y7ndbrXHg= -github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.8/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w= -github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.8 h1:lmseSULUmuVycRBJ6DVH86eFOQhHz32hN8mfxF7z+0w= -github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.8/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/duckdb/duckdb-go-bindings v0.1.19 h1:t8fwgKlr/5BEa5TJzvo3Vdr3yAgoYiR7L/TqyMuUQ2k= +github.com/duckdb/duckdb-go-bindings v0.1.19/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc= +github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.19 h1:CdNZfRcFUFxI4Q+1Tu4TBFln9tkIn6bDwVwh9LeEsoo= +github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.19/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA= +github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.19 h1:mVijr3WFz3TXZLtAm5Hb6qEnstacZdFI5QQNuE9R2QQ= +github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.19/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI= +github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.19 h1:jhchUY24T5bQLOwGyK0BzB6+HQmsRjAbgUZDKWo4ajs= +github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.19/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk= +github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.19 h1:CFcH+Bze2OgTaTLM94P3gJ554alnCCDnt1BH/nO8RJ8= +github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.19/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w= +github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.19 h1:x/8t04sgCVU8JL0XLUZWmC1FAX13ZjM58EmsyPjvrvY= +github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.19/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elastic/go-grok v0.3.1 h1:WEhUxe2KrwycMnlvMimJXvzRa7DoByJB4PVUIE1ZD/U= @@ -343,24 +802,44 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= +github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= +github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= -github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gertd/go-pluralize v0.2.1 h1:M3uASbVjMnTsPb0PNqg+E/24Vwigyo/tvyMTtAlLgiA= github.com/gertd/go-pluralize v0.2.1/go.mod h1:rbYaKDbsXxmRfr8uygAEKhOWsjyrrqrkHVpZvoOp8zk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.6.0 h1:w2hPNtoehvJIxR00Vb4xX94qHQi/ApZfX+nBE2Cjio8= @@ -372,13 +851,19 @@ github.com/go-git/go-git/v5 v5.13.0/go.mod h1:Wjo7/JyVKtQgUNdXYXIepzWfJQkUEIGvkv github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI= +github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= +github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -391,15 +876,19 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/goccy/go-yaml v1.11.2 h1:joq77SxuyIs9zzxEjgyLBugMQ9NEgTWxXfz2wVqwAaQ= -github.com/goccy/go-yaml v1.11.2/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= +github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY= +github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -431,15 +920,18 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/flatbuffers v25.1.24+incompatible h1:4wPqL3K7GzBd1CwyhSd3usxLKOaJN/AC6puCca6Jm7o= -github.com/google/flatbuffers v25.1.24+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= +github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -455,13 +947,15 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -473,6 +967,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -481,8 +976,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -490,8 +985,10 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= -github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= -github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -501,9 +998,12 @@ github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99 github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= -github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= -github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= +github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= @@ -511,13 +1011,15 @@ github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tf github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.7.5 h1:dT58k9hQ/vbxNMwoI5+xFYAJuv6152UNvdHokfI5wE4= -github.com/hashicorp/go-getter v1.7.5/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/hashicorp/go-getter v1.7.9 h1:G9gcjrDixz7glqJ+ll5IWvggSBR+R0B54DSRt4qfdC4= +github.com/hashicorp/go-getter v1.7.9/go.mod h1:dyFCmT1AQkDfOIt9NH8pw9XBDqNrIKJT5ylbpi7zPNE= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= @@ -535,8 +1037,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= -github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= +github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE= +github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM= github.com/hashicorp/terraform-registry-address v0.2.1 h1:QuTf6oJ1+WSflJw6WYOHhLgwUiQ0FrROpHPYFtwTYWM= github.com/hashicorp/terraform-registry-address v0.2.1/go.mod h1:BSE9fIFzp0qWsJUUyGquo4ldV9k2n+psif6NYkBRS3Y= github.com/hashicorp/terraform-svchost v0.1.1 h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S52uzrw4x0jKQ= @@ -545,6 +1047,7 @@ github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f h1:7LYC+Yfkj3CTRcShK0KOL/w6iTiKyqqBA9a41Wnggw8= github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -584,17 +1087,25 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/karrick/gows v0.3.0 h1:/FGSuBiJMUqNOJPsAdLvHFg7RnkFoWBS8USpdco5ONQ= github.com/karrick/gows v0.3.0/go.mod h1:kdZ/jfdo8yqKYn+BMjBkhP+/oRKUABR1abaomzRi/n8= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= -github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -607,16 +1118,19 @@ github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczG github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/marcboeker/go-duckdb/arrowmapping v0.0.6 h1:FaNX2JP4pKw7Xh2rMBCCvqWIafhX3nSXrUffexNRB68= -github.com/marcboeker/go-duckdb/arrowmapping v0.0.6/go.mod h1:WjLM334CLZux/OtAeF0DT2n9LyNqquqT3EhCHQcflNk= -github.com/marcboeker/go-duckdb/mapping v0.0.6 h1:Y+nHQDHXqo78i8MM4UP7qVmFgTAofbdvpUdRdxJXjSk= -github.com/marcboeker/go-duckdb/mapping v0.0.6/go.mod h1:k1lwBZvSza+RSpuA1kcMS/vxlNuqqFynoDef/clDD2M= -github.com/marcboeker/go-duckdb/v2 v2.1.0 h1:mhAEwy+Ut9Iji+QvyjkB86HhhC/r/H0RRKpkwfANu88= -github.com/marcboeker/go-duckdb/v2 v2.1.0/go.mod h1:W76KqN7EWTm8kpU2irA0V4f1R+6QEt3uLUVZ3wAtZ7M= +github.com/marcboeker/go-duckdb/arrowmapping v0.0.19 h1:kMxJBauR2+jwRoSFjiL/DysQtKRBCkNSLZz7GUvEG8A= +github.com/marcboeker/go-duckdb/arrowmapping v0.0.19/go.mod h1:19JWoch6I++gIrWUz1MLImIoFGri9yL54JaWn/Ujvbo= +github.com/marcboeker/go-duckdb/mapping v0.0.19 h1:xZ7LCyFZZm/4X631lOZY74p3QHINMnWJ+OakKw5d3Ao= +github.com/marcboeker/go-duckdb/mapping v0.0.19/go.mod h1:Kz9xYOkhhkgCaGgAg34ciKaks9ED2V7BzHzG6dnVo/o= +github.com/marcboeker/go-duckdb/v2 v2.4.0 h1:XztCDzB0fYvokiVer1myuFX4QvOdnicdTPRp4D+x2Ok= +github.com/marcboeker/go-duckdb/v2 v2.4.0/go.mod h1:qpTBjqtTS5+cfD3o2Sl/W70cmxKj6zhjtvVxs1Wuy7k= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -624,7 +1138,6 @@ github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -639,6 +1152,7 @@ github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= @@ -651,8 +1165,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= -github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= @@ -681,34 +1195,53 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pkg/term v1.1.0 h1:xIAAdCMh3QIAy+5FrE8Ad8XoDhEU4ufwbaSozViP9kk= github.com/pkg/term v1.1.0/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/satyrius/gonx v1.4.0 h1:F3uxif5Yx6FBzdQAh79bHQK6CTJugOcN0w0Z8azQuQg= github.com/satyrius/gonx v1.4.0/go.mod h1:+r8KNe5d2tjkZU+DfhERo0G6KxkGih+1qYF6tqLHwvk= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02 h1:v9ezJDHA1XGxViAUSIoO/Id7Fl63u6d0YmsAm+/p2hs= github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02/go.mod h1:RF16/A3L0xSa0oSERcnhd8Pu3IXSDZSK2gmGIMsttFE= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= @@ -722,22 +1255,29 @@ github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/stevenle/topsort v0.2.0 h1:LLWgtp34HPX6/RBDRS0kElVxGOTzGBLI1lSAa5Lb46k= github.com/stevenle/topsort v0.2.0/go.mod h1:ck2WG2/ZrOr6dLApQ/5Xrqy5wv3T0qhKYWE7r9tkibc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -747,38 +1287,40 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= +github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/thediveo/enumflag/v2 v2.0.5 h1:VJjvlAqUb6m6mxOrB/0tfBJI0Kvi9wJ8ulh38xK87i8= github.com/thediveo/enumflag/v2 v2.0.5/go.mod h1:0NcG67nYgwwFsAvoQCmezG0J0KaIxZ0f7skg9eLq1DA= github.com/thediveo/success v1.0.1 h1:NVwUOwKUwaN8szjkJ+vsiM2L3sNBFscldoDJ2g2tAPg= github.com/thediveo/success v1.0.1/go.mod h1:AZ8oUArgbIsCuDEWrzWNQHdKnPbDOLQsWOFj9ynwLt0= -github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= -github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= -github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= -github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tkrajina/go-reflector v0.5.8 h1:yPADHrwmUbMq4RGEyaOUpz2H90sRsETNVpjzo3DLVQQ= github.com/tkrajina/go-reflector v0.5.8/go.mod h1:ECbqLgccecY5kPmPmXg1MrHW585yMcDkVl6IvJe64T4= -github.com/turbot/go-kit v1.2.0 h1:4pBDu2LGoqF2y6ypL4xJjqlDW0BkUw3IIDlyHkU0O88= -github.com/turbot/go-kit v1.2.0/go.mod h1:1xmRuQ0cn/10QUMNLNOAFIqN8P6Rz5s3VLT8mkN3nF8= +github.com/turbot/go-kit v1.3.0 h1:6cIYPAO5hO9fG7Zd5UBC4Ch3+C6AiiyYS0UQnrUlTV0= +github.com/turbot/go-kit v1.3.0/go.mod h1:piKJMYCF8EYmKf+D2B78Csy7kOHGmnQVOWingtLKWWQ= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50 h1:zs87uA6QZsYLk4RRxDOIxt8ro/B2V6HzoMWm05Lo7ao= github.com/turbot/go-prompt v0.2.6-steampipe.0.0.20221028122246-eb118ec58d50/go.mod h1:vFnjEGDIIA/Lib7giyE4E9c50Lvl8j0S+7FVlAwDAVw= -github.com/turbot/pipe-fittings/v2 v2.3.3 h1:7nz5MQah4++qL4zLP7eRNyhiwwm9eXhdwbiyGyYSqfA= -github.com/turbot/pipe-fittings/v2 v2.3.3/go.mod h1:wEoN4EseMTXophNlpOe740rAC9Jg0JhGRt5QM5R2ss8= +github.com/turbot/pipe-fittings/v2 v2.7.0 h1:eCmpMNlVtV3AxOzsn8njE3O6aoHc74WVAHOntia2hqY= +github.com/turbot/pipe-fittings/v2 v2.7.0/go.mod h1:V619+tgfLaqoEXFDNzA2p24TBZVf4IkDL9FDLQecMnE= github.com/turbot/pipes-sdk-go v0.12.0 h1:esbbR7bALa5L8n/hqroMPaQSSo3gNM/4X0iTmHa3D6U= github.com/turbot/pipes-sdk-go v0.12.0/go.mod h1:Mb+KhvqqEdRbz/6TSZc2QWDrMa5BN3E4Xw+gPt2TRkc= -github.com/turbot/tailpipe-plugin-core v0.2.1 h1:rwnZPZxUzzKYQseq2RF6r2jMwirjCmG1J75z0T6uD6Y= -github.com/turbot/tailpipe-plugin-core v0.2.1/go.mod h1:OLuKWsJGF3JSvq8XVXBiz56ImobmVRJaweSbkXdwzcI= -github.com/turbot/tailpipe-plugin-sdk v0.3.1 h1:cC0yWMXSC/W8x/3qVPbyBXjpOcgc66vTiT1UtSmq5G8= -github.com/turbot/tailpipe-plugin-sdk v0.3.1/go.mod h1:Ztpp5tvSAu4KTcNHwIrDD9VQUGOOwcpeOwbXz8JQS0k= +github.com/turbot/tailpipe-plugin-core v0.2.10 h1:2+B7W4hzyS/pBr1y5ns9w84piWGq/x+WdCUjyPaPreQ= +github.com/turbot/tailpipe-plugin-core v0.2.10/go.mod h1:dHzPUR1p5GksSvDqqEeZEvvJX6wTEwK/ZDev//9nSLw= +github.com/turbot/tailpipe-plugin-sdk v0.9.3 h1:JpGpGPwehqdXnRO3aqkQTpd96Vx2blY+AkXP8lYB32g= +github.com/turbot/tailpipe-plugin-sdk v0.9.3/go.mod h1:Egojp0j7+th/4Bh6muMuF6aZa5iE3MuiJ4pzBo0J2mg= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7 h1:qDMxFVd8Zo0rIhnEBdCIbR+T6WgjwkxpFZMN8zZmmjg= github.com/turbot/terraform-components v0.0.0-20231213122222-1f3526cab7a7/go.mod h1:5hzpfalEjfcJWp9yq75/EZoEu2Mzm34eJAPm3HOW2tw= -github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.14 h1:uv/0Bq533iFdnMHZdRBTOlaNMdb1+ZxXIlHDZHIHcvg= +github.com/ulikunitz/xz v0.5.14/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -786,17 +1328,20 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= -github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8= -github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= -github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY34Wul7O/MSKey3txpPYyCqVO5ZyceuQJEI= -github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zclconf/go-cty v1.16.3 h1:osr++gw2T61A8KVYHoQiFbFd1Lh3JOCXc/jFLJXKTxk= +github.com/zclconf/go-cty v1.16.3/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= github.com/zclconf/go-cty-yaml v1.0.3 h1:og/eOQ7lvA/WWhHGFETVWNduJM7Rjsv2RRpx1sdFMLc= github.com/zclconf/go-cty-yaml v1.0.3/go.mod h1:9YLUH4g7lOhVWqUbctnVlZ5KLpg7JAprQNgxSZ1Gyxs= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -808,21 +1353,29 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= -go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= -go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= -go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= -go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= -go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= -go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= -go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= -go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= -go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0 h1:PB3Zrjs1sG1GBX51SXyTSoOTqcDglmsk7nT6tkKPb/k= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= @@ -830,25 +1383,48 @@ go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTV golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= -golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc= -golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= +golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -872,9 +1448,17 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -905,11 +1489,14 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -920,10 +1507,23 @@ golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= -golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -947,10 +1547,14 @@ golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7Lm golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -964,9 +1568,15 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1007,11 +1617,14 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1020,10 +1633,12 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1042,15 +1657,39 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= -golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1060,16 +1699,31 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1082,6 +1736,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1110,18 +1765,26 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= -golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1132,8 +1795,16 @@ golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNq golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= -gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1181,9 +1852,18 @@ google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaE google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI= -google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= +google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.230.0 h1:2u1hni3E+UXAXrONrrkfWpi/V6cyKVAbfGVeGtC3OxM= +google.golang.org/api v0.230.0/go.mod h1:aqvtoMk7YkiXx+6U12arQFExiRV9D/ekvMCwCd/TksQ= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1226,7 +1906,9 @@ google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1259,6 +1941,7 @@ google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= @@ -1291,13 +1974,41 @@ google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53B google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= -google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCeFn7MoLy+wBDQZ1UQIJD4UNM1Qvkg= -google.golang.org/genproto v0.0.0-20240722135656-d784300faade/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= -google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U= -google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 h1:XVhgTWWV3kGQlwJHR3upFWZeTsei6Oks1apkZSeonIE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201204527-e3fa12d562f3/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230123190316-2c411cf9d197/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= +google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb h1:ITgPrl429bc6+2ZraNSzMDk3I95nmQln2fuPstKwFDE= +google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:sAo5UzpjUwgFBCzupwhcLcxHVDK7vG5IqI30YnwX2eE= +google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 h1:FiusG7LWj+4byqhbvmB+Q93B/mOxJLN2DTozDuZm4EU= +google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:kXqgZtrWaf6qS3jZOCnCH7WYfrvFjkC51bM8fz3RsCA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1324,6 +2035,7 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= @@ -1333,8 +2045,13 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= -google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1351,8 +2068,11 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= -google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -1378,9 +2098,45 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= +modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= +modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= +modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= +modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= +modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= +modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= +modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/internal/cmdconfig/cmd_hooks.go b/internal/cmdconfig/cmd_hooks.go index 7c8a3d3b..2272aa00 100644 --- a/internal/cmdconfig/cmd_hooks.go +++ b/internal/cmdconfig/cmd_hooks.go @@ -13,7 +13,9 @@ import ( "github.com/turbot/pipe-fittings/v2/app_specific" "github.com/turbot/pipe-fittings/v2/cmdconfig" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/error_helpers" + "github.com/turbot/pipe-fittings/v2/contexthelpers" + perror_helpers "github.com/turbot/pipe-fittings/v2/error_helpers" + "github.com/turbot/pipe-fittings/v2/filepaths" pparse "github.com/turbot/pipe-fittings/v2/parse" "github.com/turbot/pipe-fittings/v2/task" @@ -21,8 +23,8 @@ import ( "github.com/turbot/pipe-fittings/v2/workspace_profile" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" - "github.com/turbot/tailpipe/internal/database" "github.com/turbot/tailpipe/internal/logger" + "github.com/turbot/tailpipe/internal/migration" "github.com/turbot/tailpipe/internal/parse" "github.com/turbot/tailpipe/internal/plugin" ) @@ -47,16 +49,11 @@ func preRunHook(cmd *cobra.Command, args []string) error { ew := initGlobalConfig(ctx) // display any warnings ew.ShowWarnings() - // TODO #errors sort exit code https://github.com/turbot/tailpipe/issues/106 // check for error - error_helpers.FailOnError(ew.Error) + perror_helpers.FailOnError(ew.Error) // pump in the initial set of logs (AFTER we have loaded the config, which may specify log level) - // this will also write out the Execution ID - enabling easy filtering of logs for a single execution - // we need to do this since all instances will log to a single file and logs will be interleaved - slog.Info("Tailpipe CLI", - "app version", viper.GetString("main.version"), - "log level", os.Getenv(app_specific.EnvLogLevel)) + displayStartupLog() // runScheduledTasks skips running tasks if this instance is the plugin manager waitForTasksChannel = runScheduledTasks(ctx, cmd, args) @@ -64,7 +61,47 @@ func preRunHook(cmd *cobra.Command, args []string) error { // set the max memory if specified setMemoryLimit() - return nil + // create cancel context and set back on command + baseCtx := cmd.Context() + ctx, cancel := context.WithCancel(baseCtx) + + // start the cancel handler to call cancel on interrupt signals + contexthelpers.StartCancelHandler(cancel) + cmd.SetContext(ctx) + + // migrate legacy data to DuckLake: + // Prior to Tailpipe v0.7.0 we stored data as native Parquet files alongside a tailpipe.db + // (DuckDB) that defined SQL views. From v0.7.0 onward Tailpipe uses DuckLake, which + // introduces a metadata database (metadata.sqlite). We run a one-time migration here to + // move existing user data into DuckLakeā€™s layout so it can be queried and managed via + // the new metadata model. + // start migration + err := migration.MigrateDataToDucklake(cmd.Context()) + if err != nil { + // we do not want Cobra usage errors for migration errors - suppress + + // suppress usage and error printing for migration errors + cmd.SilenceUsage = true + // for cancelled errors, also silence the error message + if perror_helpers.IsCancelledError(err) { + cmd.SilenceErrors = true + } + } + + // return (possibly nil) error from migration + return err +} + +func displayStartupLog() { + slog.Info("Tailpipe CLI", + "app version", viper.GetString("main.version"), + "log level", os.Getenv(app_specific.EnvLogLevel)) + + // log resource limits + slog.Info("Resource limits", + "max CLI memory (mb)", viper.GetInt64(pconstants.ArgMemoryMaxMb), + "max plugin memory (mb)", viper.GetInt64(pconstants.ArgMemoryMaxMbPlugin), + "max temp dir size (mb)", viper.GetInt64(pconstants.ArgTempDirMaxMb)) } // postRunHook is a function that is executed after the PostRun of every command handler @@ -88,6 +125,7 @@ func postRunHook(_ *cobra.Command, _ []string) error { func setMemoryLimit() { maxMemoryBytes := viper.GetInt64(pconstants.ArgMemoryMaxMb) * 1024 * 1024 if maxMemoryBytes > 0 { + slog.Info("Setting CLI memory limit", "max memory (mb)", maxMemoryBytes/(1024*1024)) // set the max memory debug.SetMemoryLimit(maxMemoryBytes) } @@ -119,7 +157,7 @@ func runScheduledTasks(ctx context.Context, cmd *cobra.Command, args []string) c } // initConfig reads in config file and ENV variables if set. -func initGlobalConfig(ctx context.Context) error_helpers.ErrorAndWarnings { +func initGlobalConfig(ctx context.Context) perror_helpers.ErrorAndWarnings { utils.LogTime("cmdconfig.initGlobalConfig start") defer utils.LogTime("cmdconfig.initGlobalConfig end") @@ -136,20 +174,14 @@ func initGlobalConfig(ctx context.Context) error_helpers.ErrorAndWarnings { // load workspace profile from the configured install dir loader, err := cmdconfig.GetWorkspaceProfileLoader[*workspace_profile.TailpipeWorkspaceProfile](parseOpts...) if err != nil { - return error_helpers.NewErrorsAndWarning(err) + return perror_helpers.NewErrorsAndWarning(err) } config.GlobalWorkspaceProfile = loader.GetActiveWorkspaceProfile() // create the required data and internal folder for this workspace if needed err = config.GlobalWorkspaceProfile.EnsureWorkspaceDirs() if err != nil { - return error_helpers.NewErrorsAndWarning(err) - } - - // ensure we have a database file for this workspace - err = database.EnsureDatabaseFile(ctx) - if err != nil { - return error_helpers.NewErrorsAndWarning(err) + return perror_helpers.NewErrorsAndWarning(err) } var cmd = viper.Get(pconstants.ConfigKeyActiveCommand).(*cobra.Command) @@ -170,33 +202,17 @@ func initGlobalConfig(ctx context.Context) error_helpers.ErrorAndWarnings { // NOTE: if this installed the core plugin, the plugin version file will be updated and the updated file returned pluginVersionFile, err := plugin.EnsureCorePlugin(ctx) if err != nil { - return error_helpers.NewErrorsAndWarning(err) + return perror_helpers.NewErrorsAndWarning(err) } // load the connection config and HCL options (passing plugin versions tailpipeConfig, loadConfigErrorsAndWarnings := parse.LoadTailpipeConfig(pluginVersionFile) - if loadConfigErrorsAndWarnings.Error != nil { - return loadConfigErrorsAndWarnings - } + if loadConfigErrorsAndWarnings.Error == nil { + // store global config + config.GlobalConfig = tailpipeConfig - if loadConfigErrorsAndWarnings.Warnings != nil { - for _, warning := range loadConfigErrorsAndWarnings.Warnings { - error_helpers.ShowWarning(warning) - } } - // store global config - config.GlobalConfig = tailpipeConfig - - // now validate all config values have appropriate values - return validateConfig() -} - -// now validate config values have appropriate values -func validateConfig() error_helpers.ErrorAndWarnings { - var res = error_helpers.ErrorAndWarnings{} - - // TODO #config validate - return res + return loadConfigErrorsAndWarnings } diff --git a/internal/cmdconfig/diagnostics.go b/internal/cmdconfig/diagnostics.go index b40a97ec..ea2d26fa 100644 --- a/internal/cmdconfig/diagnostics.go +++ b/internal/cmdconfig/diagnostics.go @@ -9,8 +9,8 @@ import ( "strings" "github.com/spf13/viper" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/tailpipe/internal/constants" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" ) // DisplayConfig prints all config set via WorkspaceProfile or HCL options @@ -51,7 +51,7 @@ func DisplayConfig() { sort.Strings(lines) var b strings.Builder - b.WriteString("\n================\nSteampipe Config\n================\n\n") + b.WriteString("\n================\nTailpipe Config\n================\n\n") for _, line := range lines { b.WriteString(line) diff --git a/internal/cmdconfig/mappings.go b/internal/cmdconfig/mappings.go index e79f1a8d..82cd16f5 100644 --- a/internal/cmdconfig/mappings.go +++ b/internal/cmdconfig/mappings.go @@ -13,9 +13,6 @@ func configDefaults(cmd *cobra.Command) map[string]any { defs := map[string]any{ // global general options pconstants.ArgUpdateCheck: true, - - // memory - pconstants.ArgMemoryMaxMb: 1024, } cmdSpecificDefaults, ok := cmdSpecificDefaults()[cmd.Name()] @@ -50,6 +47,7 @@ func envMappings() map[string]cmdconfig.EnvMapping { app_specific.EnvConfigPath: {ConfigVar: []string{pconstants.ArgConfigPath}, VarType: cmdconfig.EnvVarTypeString}, app_specific.EnvQueryTimeout: {ConfigVar: []string{pconstants.ArgDatabaseQueryTimeout}, VarType: cmdconfig.EnvVarTypeInt}, app_specific.EnvMemoryMaxMbPlugin: {ConfigVar: []string{pconstants.ArgMemoryMaxMbPlugin}, VarType: cmdconfig.EnvVarTypeInt}, + app_specific.EnvTempDirMaxMb: {ConfigVar: []string{pconstants.ArgTempDirMaxMb}, VarType: cmdconfig.EnvVarTypeInt}, constants.EnvPluginStartTimeout: {ConfigVar: []string{pconstants.ArgPluginStartTimeout}, VarType: cmdconfig.EnvVarTypeInt}, } } diff --git a/internal/collector/collector.go b/internal/collector/collector.go index 7e89c7f8..0d04b0a5 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -13,28 +13,33 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/spf13/viper" pconstants "github.com/turbot/pipe-fittings/v2/constants" + "github.com/turbot/pipe-fittings/v2/statushooks" + "github.com/turbot/tailpipe-plugin-sdk/events" sdkfilepaths "github.com/turbot/tailpipe-plugin-sdk/filepaths" - "github.com/turbot/tailpipe-plugin-sdk/grpc/proto" "github.com/turbot/tailpipe-plugin-sdk/row_source" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/filepaths" - "github.com/turbot/tailpipe/internal/parquet" + localfilepaths "github.com/turbot/tailpipe/internal/filepaths" + "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/tailpipe/internal/plugin" ) const eventBufferSize = 100 type Collector struct { - Events chan *proto.Event + // this buffered channel is used to asynchronously process events from the plugin + // our Notify function will send events to this channel + Events chan events.Event + // the plugin manager is responsible for managing the plugin lifecycle and brokering + // communication between the plugin and the collector pluginManager *plugin.PluginManager // the partition to collect partition *config.Partition - // the execution + // the execution is used to manage the state of the collection execution *execution - - parquetConvertor *parquet.Converter + // the parquet convertor is used to convert the JSONL files to parquet + parquetConvertor *database.Converter // the current plugin status - used to update the spinner status status @@ -44,23 +49,32 @@ type Collector struct { // the subdirectory of ~/.tailpipe/internal which will be used to store all file data for this collection // this will have the form ~/.tailpipe/internal/collection// collectionTempDir string - + // the path to the JSONL files - the plugin will write to this path sourcePath string + // database connection + db *database.DuckDb + // bubble tea app app *tea.Program cancel context.CancelFunc } +// New creates a new collector func New(pluginManager *plugin.PluginManager, partition *config.Partition, cancel context.CancelFunc) (*Collector, error) { // get the temp data dir for this collection // - this is located in ~/.turbot/internal/collection// // first clear out any old collection temp dirs - filepaths.CleanupCollectionTempDirs() - collectionTempDir := filepaths.EnsureCollectionTempDir() + // get the collection directory for this workspace + collectionDir := config.GlobalWorkspaceProfile.GetCollectionDir() + filepaths.CleanupPidTempDirs(collectionDir) + // then create a new collection temp dir + collectionTempDir := localfilepaths.EnsureCollectionTempDir() + + // create the collector c := &Collector{ - Events: make(chan *proto.Event, eventBufferSize), + Events: make(chan events.Event, eventBufferSize), pluginManager: pluginManager, collectionTempDir: collectionTempDir, partition: partition, @@ -77,24 +91,49 @@ func New(pluginManager *plugin.PluginManager, partition *config.Partition, cance } c.sourcePath = sourcePath + // create the DuckDB connection + // load inet extension in addition to the DuckLake extension + db, err := database.NewDuckDb( + database.WithDuckDbExtensions(pconstants.DuckDbExtensions), + database.WithDuckLake(), + ) + + if err != nil { + return nil, fmt.Errorf("failed to create DuckDB connection: %w", err) + } + c.db = db + return c, nil } +// Close closes the collector +// - closes the events channel +// - closes the parquet convertor +// - removes the JSONL path +// - removes the collection temp dir func (c *Collector) Close() { close(c.Events) - if c.parquetConvertor != nil { - c.parquetConvertor.Close() - } - // if inbox path is empty, remove it (ignore errors) _ = os.Remove(c.sourcePath) // delete the collection temp dir _ = os.RemoveAll(c.collectionTempDir) + + // close the tea app + if c.app != nil { + c.app.Quit() + } } -func (c *Collector) Collect(ctx context.Context, fromTime time.Time) (err error) { +// Collect asynchronously starts the collection process +// - creates a new execution +// - tells the plugin manager to start collecting +// - validates the schema returned by the plugin +// - starts the collection UI +// - creates a parquet writer, which will process the JSONL files as they are written +// - starts listening to plugin events +func (c *Collector) Collect(ctx context.Context, fromTime, toTime time.Time, overwrite bool) (err error) { if c.execution != nil { return errors.New("collection already in progress") } @@ -108,62 +147,106 @@ func (c *Collector) Collect(ctx context.Context, fromTime time.Time) (err error) } }() - // create the execution _before_ calling the plugin to ensure it is ready to receive the started event - c.execution = newExecution(c.partition) - - // tell plugin to start collecting - collectResponse, err := c.pluginManager.Collect(ctx, c.partition, fromTime, c.collectionTempDir) - if err != nil { - return err + var collectResponse *plugin.CollectResponse + // is this is a synthetic partition? + if c.partition.SyntheticMetadata != nil { + if collectResponse, err = c.doCollectSynthetic(ctx, fromTime, toTime, overwrite); err != nil { + return err + } + } else { + if collectResponse, err = c.doCollect(ctx, fromTime, toTime, overwrite); err != nil { + return err + } } - resolvedFromTime := collectResponse.FromTime - // now set the execution id - c.execution.id = collectResponse.ExecutionId - // validate the schema returned by the plugin - err = collectResponse.Schema.Validate() - if err != nil { + if err = collectResponse.Schema.Validate(); err != nil { err := fmt.Errorf("table '%s' returned invalid schema: %w", c.partition.TableName, err) // set execution to error c.execution.done(err) // and return error return err } + // determine the time to start collecting from + resolvedFromTime := collectResponse.FromTime + + // if we are overwriting, we need to delete any existing data in the partition + if overwrite { + // show spinner while deleting the partition + spinner := statushooks.NewStatusSpinnerHook() + spinner.SetStatus(fmt.Sprintf("Deleting partition %s", c.partition.TableName)) + spinner.Show() + err := c.deletePartitionData(ctx, resolvedFromTime.Time, toTime) + spinner.Hide() + if err != nil { + // set execution to error + c.execution.done(err) + // and return error + return fmt.Errorf("failed to delete partition data: %w", err) + } + } // display the progress UI - err = c.showCollectionStatus(resolvedFromTime) + err = c.showCollectionStatus(resolvedFromTime, toTime) if err != nil { return err } - // if there is a from time, add a filter to the partition - this will be used by the parquet writer - if !resolvedFromTime.Time.IsZero() { - // NOTE: handle null timestamp so we get a validation error for null timestamps, rather than excluding the row - c.partition.AddFilter(fmt.Sprintf("tp_timestamp is null or tp_timestamp >= '%s'", resolvedFromTime.Time.Format("2006-01-02T15:04:05"))) - } + // if we have a from or to time, add filters to the partition + c.addTimeRangeFilters(resolvedFromTime, toTime) // create a parquet writer - parquetConvertor, err := parquet.NewParquetConverter(ctx, cancel, c.execution.id, c.partition, c.sourcePath, collectResponse.Schema, c.updateRowCount) + parquetConvertor, err := database.NewParquetConverter(ctx, cancel, c.execution.id, c.partition, c.sourcePath, collectResponse.Schema, c.updateRowCount, c.db) if err != nil { return fmt.Errorf("failed to create parquet writer: %w", err) } c.parquetConvertor = parquetConvertor - // start listening to plugin event - c.listenToEventsAsync(ctx) + // start listening to plugin events asynchronously + go c.listenToEvents(ctx) return nil } +func (c *Collector) doCollect(ctx context.Context, fromTime time.Time, toTime time.Time, overwrite bool) (*plugin.CollectResponse, error) { + // create the execution + // NOTE: create _before_ calling the plugin to ensure it is ready to receive the started event + c.execution = newExecution(c.partition) + + // tell plugin to start collecting + collectResponse, err := c.pluginManager.Collect(ctx, c.partition, fromTime, toTime, overwrite, c.collectionTempDir) + if err != nil { + return nil, err + } + + // _now_ set the execution id + c.execution.id = collectResponse.ExecutionId + return collectResponse, nil +} + +// addTimeRangeFilters adds filters to the partition based on the from and to time +func (c *Collector) addTimeRangeFilters(resolvedFromTime *row_source.ResolvedFromTime, toTime time.Time) { + // if there is a from time, add a filter to the partition - this will be used by the parquet writer + if !resolvedFromTime.Time.IsZero() { + // NOTE: handle null timestamp so we get a validation error for null timestamps, rather than excluding the row + c.partition.AddFilter(fmt.Sprintf("(tp_timestamp is null or tp_timestamp >= '%s')", resolvedFromTime.Time.Format("2006-01-02T15:04:05"))) + } + // if to time was set as arg, add that filter as well + if viper.IsSet(pconstants.ArgTo) { + // NOTE: handle null timestamp so we get a validation error for null timestamps, rather than excluding the row + c.partition.AddFilter(fmt.Sprintf("(tp_timestamp is null or tp_timestamp < '%s')", toTime.Format("2006-01-02T15:04:05"))) + } +} + // Notify implements observer.Observer // send an event down the channel to be picked up by the handlePluginEvent goroutine -func (c *Collector) Notify(event *proto.Event) { +func (c *Collector) Notify(_ context.Context, event events.Event) error { // only send the event if the execution is not complete - this is to handle the case where it has // terminated with an error, causing the collector to close, closing the channel if !c.execution.complete() { c.Events <- event } + return nil } // WaitForCompletion waits for our execution to have state ExecutionState_COMPLETE @@ -174,20 +257,25 @@ func (c *Collector) WaitForCompletion(ctx context.Context) error { return c.execution.waitForCompletion(ctx) } -// Compact compacts the parquet files +// Compact compacts the parquet files so that for each date folder +// (the lowest level of the partition) there is only one parquet file func (c *Collector) Compact(ctx context.Context) error { slog.Info("Compacting parquet files") c.updateApp(AwaitingCompactionMsg{}) - updateAppCompactionFunc := func(compactionStatus parquet.CompactionStatus) { + updateAppCompactionFunc := func(status database.CompactionStatus) { c.statusLock.Lock() defer c.statusLock.Unlock() - c.status.UpdateCompactionStatus(&compactionStatus) + c.status.compactionStatus = &status c.updateApp(CollectionStatusUpdateMsg{status: c.status}) } - partitionPattern := parquet.NewPartitionPattern(c.partition) - err := parquet.CompactDataFiles(ctx, updateAppCompactionFunc, partitionPattern) + partitionPattern := database.NewPartitionPattern(c.partition) + + // NOTE: we DO NOT reindex when compacting after collection + reindex := false + err := database.CompactDataFiles(ctx, c.db, updateAppCompactionFunc, reindex, &partitionPattern) + if err != nil { return fmt.Errorf("failed to compact data files: %w", err) } @@ -203,119 +291,54 @@ func (c *Collector) Completed() { // if we suppressed progress display, we should write the summary if !viper.GetBool(pconstants.ArgProgress) { - fmt.Fprint(os.Stdout, c.StatusString()) //nolint:forbidigo // we are writing to stdout + _, _ = fmt.Fprint(os.Stdout, c.StatusString()) //nolint:forbidigo // UI output } } -// handlePluginEvent handles an event from a plugin -func (c *Collector) handlePluginEvent(ctx context.Context, e *proto.Event) { - // handlePluginEvent the event - // switch based on the struct of the event - switch e.GetEvent().(type) { - case *proto.Event_StartedEvent: - slog.Info("Event_StartedEvent", "execution", e.GetStartedEvent().ExecutionId) - c.execution.state = ExecutionState_STARTED - case *proto.Event_StatusEvent: - c.statusLock.Lock() - defer c.statusLock.Unlock() - c.status.UpdateWithPluginStatus(e.GetStatusEvent()) - c.updateApp(CollectionStatusUpdateMsg{status: c.status}) - case *proto.Event_ChunkWrittenEvent: - ev := e.GetChunkWrittenEvent() - executionId := ev.ExecutionId - chunkNumber := ev.ChunkNumber - - // log every 100 chunks - if ev.ChunkNumber%100 == 0 { - slog.Debug("Event_ChunkWrittenEvent", "execution", ev.ExecutionId, "chunk", ev.ChunkNumber) - } - - err := c.parquetConvertor.AddChunk(executionId, chunkNumber) - if err != nil { - slog.Error("failed to add chunk to parquet writer", "error", err) - c.execution.done(err) - } - case *proto.Event_CompleteEvent: - completedEvent := e.GetCompleteEvent() - slog.Info("handlePluginEvent received Event_CompleteEvent", "execution", completedEvent.ExecutionId) - - // was there an error? - if completedEvent.Error != "" { - slog.Error("execution error", "execution", completedEvent.ExecutionId, "error", completedEvent.Error) - // retrieve the execution - c.execution.done(fmt.Errorf("plugin error: %s", completedEvent.Error)) - } - // this event means all JSON files have been written - we need to wait for all to be converted to parquet - // we then combine the parquet files into a single file - - // start thread waiting for conversion to complete - // - this will wait for all parquet files to be written, and will then combine these into a single parquet file - go func() { - slog.Info("handlePluginEvent - waiting for conversions to complete", "execution", completedEvent.ExecutionId) - err := c.waitForConversions(ctx, completedEvent) - if err != nil { - slog.Error("error waiting for execution to complete", "error", err) - c.execution.done(err) - } else { - slog.Info("handlePluginEvent - conversions all complete", "execution", completedEvent.ExecutionId) - } - }() - - case *proto.Event_ErrorEvent: - // TODO #errors error events are deprecated an will only be sent for plugins not using sdk > v0.2.0 - // TODO #errors decide what (if anything) we should do with error events from old plugins https://github.com/turbot/tailpipe/issues/297 - //ev := e.GetErrorEvent() - //// for now just store errors and display at end - ////c.execution.state = ExecutionState_ERROR - ////c.execution.error = fmt.Errorf("plugin error: %s", ev.Error) - //slog.Warn("plugin error", "execution", ev.ExecutionId, "error", ev.Error) - } -} - -func (c *Collector) createTableView(ctx context.Context) error { - // so we are done writing chunks - now update the db to add a view to this data - // Open a DuckDB connection - db, err := database.NewDuckDb(database.WithDbFile(filepaths.TailpipeDbFilePath())) +// deletePartitionData deletes all parquet files in the partition between the fromTime and toTime +func (c *Collector) deletePartitionData(ctx context.Context, fromTime, toTime time.Time) error { + slog.Info("Deleting parquet files after the from time", "partition", c.partition.Name, "from", fromTime) + _, err := database.DeletePartition(ctx, c.partition, fromTime, toTime, c.db) if err != nil { - return err - } - defer db.Close() + slog.Warn("Failed to delete parquet files after the from time", "partition", c.partition.Name, "from", fromTime, "error", err) - err = database.AddTableView(ctx, c.execution.table, db) - if err != nil { - return err } - return nil + slog.Info("Completed deleting parquet files after the from time", "partition", c.partition.Name, "from", fromTime) + return err } -func (c *Collector) showCollectionStatus(resolvedFromTime *row_source.ResolvedFromTime) error { - c.status.Init(c.partition.GetUnqualifiedName(), resolvedFromTime) +func (c *Collector) showCollectionStatus(resolvedFromTime *row_source.ResolvedFromTime, toTime time.Time) error { + c.status.Init(c.partition.GetUnqualifiedName(), resolvedFromTime, toTime) + // if the progress flag is set, start the tea app to display the progress if viper.GetBool(pconstants.ArgProgress) { - return c.showTeaAppAsync() + // start the tea app asynchronously + go c.showTeaApp() + return nil } - - return c.showMinimalCollectionStatus() + // otherwise, just show a simple initial message - we will show a simple summary tat the end + return c.showMinimalCollectionStartMessage() } -func (c *Collector) showTeaAppAsync() error { +// showTeaApp starts the tea app to display the collection progress +func (c *Collector) showTeaApp() { + // create the tea app c.app = tea.NewProgram(newCollectionModel(c.status)) - - go func() { - model, err := c.app.Run() - if model.(collectionModel).cancelled { - slog.Info("Collection UI returned cancelled") - c.doCancel() - } - if err != nil { - slog.Warn("Collection UI returned error", "error", err) - c.doCancel() - } - }() - return nil + // and start it + model, err := c.app.Run() + if model.(collectionModel).cancelled { + slog.Info("Collection UI returned cancelled") + c.doCancel() + } + if err != nil { + slog.Warn("Collection UI returned error", "error", err) + c.doCancel() + } } -func (c *Collector) showMinimalCollectionStatus() error { +// showMinimalCollectionStartMessage displays a simple status message to indicate that the collection has started +// this is used when the progress flag is not set +func (c *Collector) showMinimalCollectionStartMessage() error { // display initial message initMsg := c.status.CollectionHeader() _, err := fmt.Print(initMsg) //nolint:forbidigo //desired output @@ -350,53 +373,91 @@ func (c *Collector) StatusString() string { // waitForConversions waits for the parquet writer to complete the conversion of the JSONL files to parquet // it then sets the execution state to ExecutionState_COMPLETE -func (c *Collector) waitForConversions(ctx context.Context, ce *proto.EventComplete) (err error) { - slog.Info("waitForConversions - waiting for execution to complete", "execution", ce.ExecutionId, "chunks", ce.ChunkCount, "rows", ce.RowCount) +func (c *Collector) waitForConversions(ctx context.Context, ce *events.Complete) (err error) { + slog.Info("waitForConversions - waiting for execution to complete", "execution", ce.ExecutionId, "chunks", ce.ChunksWritten, "rows", ce.RowCount) - if ce.ChunkCount == 0 && ce.RowCount == 0 { - slog.Debug("waitForConversions - no chunks/rows to write", "execution", ce.ExecutionId) - var err error - if ce.Error != "" { - slog.Warn("waitForConversions - plugin execution returned error", "execution", ce.ExecutionId, "error", ce.Error) - err = errors.New(ce.Error) - } - // mark execution as done, passing any error + // ensure we mark the execution as done + defer func() { c.execution.done(err) + }() + + // if there are no chunks or rows to write, we can just return + // (note: we know that the Complete event wil not have an error as that is handled before calling this function) + if ce.ChunksWritten == 0 && ce.RowCount == 0 { + slog.Debug("waitForConversions - no chunks/rows to write", "execution", ce.ExecutionId) return nil } - // so there was no plugin error - wait for the conversions to complete - c.parquetConvertor.WaitForConversions(ctx) + // wait for the conversions to complete + return c.parquetConvertor.WaitForConversions(ctx) +} - if err := c.createTableView(ctx); err != nil { - slog.Error("error creating table view", "error", err) - c.execution.done(err) - return err +// listenToEvents listens to the events channel and handles events +func (c *Collector) listenToEvents(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case e := <-c.Events: + c.handlePluginEvent(ctx, e) + } } +} - // mark execution as complete and record the end time - c.execution.done(err) +// handlePluginEvent handles an event from a plugin +func (c *Collector) handlePluginEvent(ctx context.Context, e events.Event) { + // handlePluginEvent the event + // switch based on the struct of the event + switch ev := e.(type) { + case *events.Started: + slog.Info("Started event", "execution", ev.ExecutionId) + c.execution.state = ExecutionState_STARTED + case *events.Status: + c.statusLock.Lock() + defer c.statusLock.Unlock() + c.status.UpdateWithPluginStatus(ev) + c.updateApp(CollectionStatusUpdateMsg{status: c.status}) + case *events.Chunk: - // if there was an error, return it - if err != nil { - return err - } + executionId := ev.ExecutionId + chunkNumber := ev.ChunkNumber - // notify the writer that the collection is complete - return nil -} + // log every 100 chunks + if ev.ChunkNumber%100 == 0 { + slog.Debug("Chunk event", "execution", ev.ExecutionId, "chunk", ev.ChunkNumber) + } -func (c *Collector) listenToEventsAsync(ctx context.Context) { - go func() { - for { - select { - case <-ctx.Done(): - return - case event := <-c.Events: - c.handlePluginEvent(ctx, event) - } + err := c.parquetConvertor.AddChunk(executionId, chunkNumber) + if err != nil { + slog.Error("failed to add chunk to parquet writer", "error", err) + c.execution.done(err) } - }() + case *events.Complete: + slog.Info("Complete event", "execution", ev.ExecutionId) + + // was there an error? + if ev.Err != nil { + slog.Error("execution error", "execution", ev.ExecutionId, "error", ev.Err) + // update the execution + c.execution.done(ev.Err) + return + } + // this event means all JSON files have been written - we need to wait for all to be converted to parquet + // we then combine the parquet files into a single file + + // start thread waiting for conversion to complete + // - this will wait for all parquet files to be written, and will then combine these into a single parquet file + slog.Info("handlePluginEvent - waiting for conversions to complete") + go func() { + err := c.waitForConversions(ctx, ev) + if err != nil { + slog.Error("error waiting for execution to complete", "error", err) + c.execution.done(err) + } else { + slog.Info("all conversions complete") + } + }() + } } func (c *Collector) doCancel() { diff --git a/internal/collector/collector_synthetic.go b/internal/collector/collector_synthetic.go new file mode 100644 index 00000000..57527ef7 --- /dev/null +++ b/internal/collector/collector_synthetic.go @@ -0,0 +1,649 @@ +package collector + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "math" + "os" + "path/filepath" + "strings" + "time" + + "bufio" + "runtime" + "sync" + + "github.com/turbot/tailpipe-plugin-sdk/events" + "github.com/turbot/tailpipe-plugin-sdk/row_source" + "github.com/turbot/tailpipe-plugin-sdk/schema" + "github.com/turbot/tailpipe-plugin-sdk/table" + "github.com/turbot/tailpipe/internal/config" + "github.com/turbot/tailpipe/internal/plugin" +) + +// doCollectSynthetic initiates synthetic data collection for testing and performance benchmarking. +// This function simulates the data collection process by generating dummy data instead of collecting from real sources. +// +// The function: +// 1. Creates an execution context to track the synthetic collection process +// 2. Builds a synthetic schema based on the number of columns specified in the partition metadata +// 3. Starts a background goroutine to generate and write synthetic data in chunks +// 4. Returns a CollectResponse that mimics what a real plugin would return +// +// This enables testing of the entire data collection pipeline without requiring actual data sources, +// making it useful for performance testing, load testing, and development/debugging scenarios. +// +// Parameters: +// - ctx: Context for cancellation and timeout handling +// - fromTime: Start time for the synthetic data (timestamps will be distributed across this range) +// - toTime: End time for the synthetic data +// - overwrite: Whether to overwrite existing data (not used in synthetic collection) +// +// Returns: +// - *plugin.CollectResponse: Response containing execution ID and schema information +// - error: Any error that occurred during initialization +func (c *Collector) doCollectSynthetic(ctx context.Context, fromTime time.Time, toTime time.Time, overwrite bool) (*plugin.CollectResponse, error) { + // Create the execution context to track the synthetic collection process + // This must be created before starting the collection goroutine to ensure proper event handling + c.execution = &execution{ + id: "synthetic", // Use "synthetic" as the execution ID + partition: c.partition.UnqualifiedName, // Full partition name for identification + table: c.partition.TableName, // Table name (always "synthetic" for synthetic partitions) + plugin: "synthetic", // Plugin name for logging and identification + state: ExecutionState_PENDING, // Initial state before collection starts + completionChan: make(chan error, 1), // Channel to signal completion or errors + } + + // Build the synthetic schema based on the number of columns specified in the partition metadata + // This creates a table schema with the specified number of columns of various types + schema := buildsyntheticchema(c.partition.SyntheticMetadata.Columns) + + // Start a background goroutine to perform the actual synthetic data generation + // This simulates the asynchronous nature of real data collection + go c.collectSynthetic(ctx, schema, fromTime, toTime) + + // Build a collect response that mimics what a real plugin would return + // This allows the synthetic collection to integrate seamlessly with the existing collection pipeline + collectResponse := &plugin.CollectResponse{ + ExecutionId: c.execution.id, // Use the execution ID for tracking + Schema: schema, // The generated synthetic schema + FromTime: &row_source.ResolvedFromTime{ + Time: fromTime, // Start time for the data collection + Source: "synthetic", // Source identifier for synthetic data + }, + } + + // Update the execution ID to match the response (in case it was modified) + c.execution.id = collectResponse.ExecutionId + return collectResponse, nil +} + +// syntheticColumnTypes defines the available column types for synthetic data generation +var syntheticColumnTypes = []struct { + Name string + SQLType string + StructFields []*schema.ColumnSchema +}{ + {"string_col", "VARCHAR", nil}, + {"int_col", "INTEGER", nil}, + {"float_col", "DOUBLE", nil}, + {"bool_col", "BOOLEAN", nil}, + {"json_col", "JSON", nil}, + {"timestamp_col", "TIMESTAMP", nil}, + {"array_col", "JSON", nil}, + {"nested_json_col", "JSON", nil}, + {"uuid_col", "VARCHAR", nil}, + {"simple_struct_col", "STRUCT", []*schema.ColumnSchema{ + { + SourceName: "id", + ColumnName: "id", + Type: "INTEGER", + Description: "Simple struct ID field", + }, + { + SourceName: "name", + ColumnName: "name", + Type: "VARCHAR", + Description: "Simple struct name field", + }, + { + SourceName: "active", + ColumnName: "active", + Type: "BOOLEAN", + Description: "Simple struct active field", + }, + }}, + {"nested_struct_col", "STRUCT", []*schema.ColumnSchema{ + { + SourceName: "metadata", + ColumnName: "metadata", + Type: "STRUCT", + StructFields: []*schema.ColumnSchema{ + { + SourceName: "created_at", + ColumnName: "created_at", + Type: "VARCHAR", + Description: "Creation timestamp", + }, + { + SourceName: "version", + ColumnName: "version", + Type: "VARCHAR", + Description: "Version string", + }, + }, + Description: "Metadata information", + }, + { + SourceName: "data", + ColumnName: "data", + Type: "STRUCT", + StructFields: []*schema.ColumnSchema{ + { + SourceName: "field1", + ColumnName: "field1", + Type: "INTEGER", + Description: "Numeric field 1", + }, + { + SourceName: "field2", + ColumnName: "field2", + Type: "VARCHAR", + Description: "String field 2", + }, + { + SourceName: "field3", + ColumnName: "field3", + Type: "BOOLEAN", + Description: "Boolean field 3", + }, + }, + Description: "Data fields", + }, + }}, + {"complex_struct_col", "STRUCT", []*schema.ColumnSchema{ + { + SourceName: "user", + ColumnName: "user", + Type: "STRUCT", + StructFields: []*schema.ColumnSchema{ + { + SourceName: "id", + ColumnName: "id", + Type: "INTEGER", + Description: "User ID", + }, + { + SourceName: "name", + ColumnName: "name", + Type: "VARCHAR", + Description: "User name", + }, + { + SourceName: "profile", + ColumnName: "profile", + Type: "STRUCT", + StructFields: []*schema.ColumnSchema{ + { + SourceName: "age", + ColumnName: "age", + Type: "INTEGER", + Description: "User age", + }, + { + SourceName: "email", + ColumnName: "email", + Type: "VARCHAR", + Description: "User email", + }, + { + SourceName: "verified", + ColumnName: "verified", + Type: "BOOLEAN", + Description: "Email verified", + }, + }, + Description: "User profile information", + }, + }, + Description: "User information", + }, + { + SourceName: "settings", + ColumnName: "settings", + Type: "STRUCT", + StructFields: []*schema.ColumnSchema{ + { + SourceName: "theme", + ColumnName: "theme", + Type: "VARCHAR", + Description: "UI theme", + }, + { + SourceName: "notifications", + ColumnName: "notifications", + Type: "BOOLEAN", + Description: "Notifications enabled", + }, + }, + Description: "User settings", + }, + }}, +} + +// ConcurrentDataGenerator handles concurrent data generation and marshaling +type ConcurrentDataGenerator struct { + numWorkers int + rowChan chan []byte + errorChan chan error + doneChan chan bool +} + +// NewConcurrentDataGenerator creates a new concurrent data generator +func NewConcurrentDataGenerator(numWorkers int) *ConcurrentDataGenerator { + return &ConcurrentDataGenerator{ + numWorkers: numWorkers, + rowChan: make(chan []byte, numWorkers*100), // Buffer for generated rows + errorChan: make(chan error, 1), + doneChan: make(chan bool, 1), + } +} + +// generateRowData generates a single row's JSON data +func generateRowData(rowIndex int, partition *config.Partition, tableSchema *schema.TableSchema, fromTime time.Time, timestampInterval time.Duration) ([]byte, error) { + // Create row map + rowMap := make(map[string]any, len(tableSchema.Columns)) + timestamp := fromTime.Add(time.Duration(rowIndex) * timestampInterval).Format("2006-01-02 15:04:05") + + // Populate row map (skip tp_index and tp_date) + for _, column := range tableSchema.Columns { + if column.ColumnName == "tp_index" || column.ColumnName == "tp_date" { + continue + } + + switch column.ColumnName { + case "tp_timestamp": + rowMap[column.ColumnName] = timestamp + case "tp_partition": + rowMap[column.ColumnName] = partition.ShortName + case "tp_table": + rowMap[column.ColumnName] = partition.TableName + default: + // Generate synthetic data for other columns + rowMap[column.ColumnName] = generateSyntheticValue(column, rowIndex) + } + } + + // Marshal to JSON + data, err := json.Marshal(rowMap) + if err != nil { + return nil, fmt.Errorf("failed to marshal row %d: %w", rowIndex, err) + } + + // Add newline + data = append(data, '\n') + return data, nil +} + +// worker generates data for a range of rows +func (cdg *ConcurrentDataGenerator) worker(startRow, endRow int, partition *config.Partition, tableSchema *schema.TableSchema, fromTime time.Time, timestampInterval time.Duration) { + for rowIndex := startRow; rowIndex < endRow; rowIndex++ { + data, err := generateRowData(rowIndex, partition, tableSchema, fromTime, timestampInterval) + if err != nil { + select { + case cdg.errorChan <- err: + default: + } + return + } + + select { + case cdg.rowChan <- data: + case <-cdg.doneChan: + return + } + } +} + +// writeOptimizedChunkToJSONLConcurrent uses multiple goroutines for data generation +func writeOptimizedChunkToJSONLConcurrent(filepath string, tableSchema *schema.TableSchema, rows int, startRowIndex int, partition *config.Partition, fromTime time.Time, timestampInterval time.Duration) error { + file, err := os.Create(filepath) + if err != nil { + return fmt.Errorf("failed to create file %s: %w", filepath, err) + } + defer file.Close() + + // Use buffered writer for better I/O performance + bufWriter := bufio.NewWriter(file) + defer bufWriter.Flush() + + // Determine number of workers (use CPU cores, but cap at reasonable number) + numWorkers := runtime.NumCPU() + if numWorkers > 8 { + numWorkers = 8 // Cap at 8 to avoid too much overhead + } + if numWorkers > rows { + numWorkers = rows // Don't create more workers than rows + } + + // Create concurrent data generator + cdg := NewConcurrentDataGenerator(numWorkers) + + // Calculate rows per worker + rowsPerWorker := rows / numWorkers + remainder := rows % numWorkers + + // Start workers + var wg sync.WaitGroup + startRow := startRowIndex + for i := 0; i < numWorkers; i++ { + endRow := startRow + rowsPerWorker + if i < remainder { + endRow++ // Distribute remainder rows + } + + wg.Add(1) + go func(start, end int) { + defer wg.Done() + cdg.worker(start, end, partition, tableSchema, fromTime, timestampInterval) + }(startRow, endRow) + + startRow = endRow + } + + // Start a goroutine to close the row channel when all workers are done + go func() { + wg.Wait() + close(cdg.rowChan) + }() + + // Write rows from channel to file + rowsWritten := 0 + for data := range cdg.rowChan { + if _, err := bufWriter.Write(data); err != nil { + close(cdg.doneChan) // Signal workers to stop + return fmt.Errorf("failed to write row %d: %w", rowsWritten, err) + } + rowsWritten++ + } + + // Check for errors + select { + case err := <-cdg.errorChan: + return fmt.Errorf("worker error: %w", err) + default: + } + + if rowsWritten != rows { + return fmt.Errorf("expected %d rows, but wrote %d", rows, rowsWritten) + } + + return nil +} + +func buildsyntheticchema(columns int) *schema.TableSchema { + // Create a basic schema with the required number of columns + // Start with required tp_ fields + s := &schema.TableSchema{ + Columns: make([]*schema.ColumnSchema, 0, columns+5), // +5 for tp_ fields (including tp_index and tp_date) + } + + // Add required tp_ fields first + tpFields := []struct { + name string + columnType string + description string + }{ + {"tp_timestamp", "TIMESTAMP", "Timestamp when the record was collected"}, + {"tp_partition", "VARCHAR", "Partition identifier"}, + {"tp_table", "VARCHAR", "Table identifier"}, + {"tp_index", "VARCHAR", "Index identifier"}, + {"tp_date", "VARCHAR", "Date identifier"}, + } + + for _, tpField := range tpFields { + column := &schema.ColumnSchema{ + SourceName: tpField.name, + ColumnName: tpField.name, + Type: tpField.columnType, + StructFields: nil, + Description: tpField.description, + Required: true, // tp_ fields are always required + NullIf: "", + Transform: "", + } + s.Columns = append(s.Columns, column) + } + + // Add the specified number of synthetic columns by cycling through the column types + for i := 0; i < columns; i++ { + // Cycle through the column types + typeIndex := i % len(syntheticColumnTypes) + baseType := syntheticColumnTypes[typeIndex] + + // Create a unique column name + columnName := fmt.Sprintf("%s_%d", baseType.Name, i) + + column := &schema.ColumnSchema{ + SourceName: columnName, + ColumnName: columnName, + Type: baseType.SQLType, + StructFields: baseType.StructFields, + Description: fmt.Sprintf("Synthetic column of type %s", baseType.SQLType), + Required: false, + NullIf: "", + Transform: "", + } + + s.Columns = append(s.Columns, column) + } + + return s +} + +// collectSynthetic generates synthetic data in chunks and writes it to JSONL files. +// This function runs in a background goroutine and simulates the data collection process +// by generating dummy data according to the synthetic partition specifications. +// +// The function: +// 1. Notifies that collection has started +// 2. Calculates timestamp intervals to distribute timestamps across the time range +// 3. Generates data in chunks according to the specified chunk size +// 4. Writes each chunk to a JSONL file using optimized concurrent writing +// 5. Respects the delivery interval to simulate real-time data flow +// 6. Sends progress events (chunk and status) to maintain the collection UI +// 7. Handles cancellation and error conditions gracefully +// 8. Notifies completion when all data has been generated +// +// Parameters: +// - ctx: Context for cancellation and timeout handling +// - tableSchema: The schema defining the structure of the synthetic data +// - fromTime: Start time for timestamp generation +// - toTime: End time for timestamp generation +func (c *Collector) collectSynthetic(ctx context.Context, tableSchema *schema.TableSchema, fromTime time.Time, toTime time.Time) { + metadata := c.partition.SyntheticMetadata + + // Set the execution state to started to indicate collection is in progress + c.execution.state = ExecutionState_STARTED + + // Notify that collection has started - this triggers the collection UI to show progress + if err := c.Notify(ctx, &events.Started{ExecutionId: c.execution.id}); err != nil { + slog.Error("failed to notify started event", "error", err) + c.execution.completionChan <- fmt.Errorf("failed to notify started event: %w", err) + return + } + + var chunkIdx int32 = 0 // Track the current chunk number + var totalRowsProcessed int64 = 0 // Track total rows processed for progress reporting + + // Calculate timestamp interval to distribute timestamps evenly across the time range + // This ensures synthetic data has realistic timestamp progression + var timestampInterval time.Duration + if metadata.Rows > 1 { + // Distribute timestamps evenly between fromTime and toTime + timestampInterval = toTime.Sub(fromTime) / time.Duration(metadata.Rows-1) + } else { + // Single row case - no interval needed + timestampInterval = 0 + } + + // Generate data in chunks according to the specified chunk size + // This allows for memory-efficient processing of large datasets + for rowCount := 0; rowCount < metadata.Rows; rowCount += metadata.ChunkSize { + t := time.Now() // Track chunk processing time for delivery interval calculation + + // Check if context is cancelled - allows for graceful shutdown + select { + case <-ctx.Done(): + c.execution.completionChan <- ctx.Err() + return + default: + } + + // Calculate the number of rows for this chunk (may be less than chunk size for the last chunk) + rows := int(math.Min(float64(metadata.Rows-rowCount), float64(metadata.ChunkSize))) + + // Generate filename for this chunk's JSONL file + filename := table.ExecutionIdToJsonlFileName(c.execution.id, chunkIdx) + filepath := filepath.Join(c.sourcePath, filename) + + // Write the chunk to JSONL file using optimized concurrent approach + // This generates synthetic data and writes it efficiently to disk + if err := writeOptimizedChunkToJSONLConcurrent(filepath, tableSchema, rows, rowCount, c.partition, fromTime, timestampInterval); err != nil { + c.execution.completionChan <- fmt.Errorf("error writing chunk to JSONL file: %w", err) + return + } + + dur := time.Since(t) // Calculate how long this chunk took to process + + // Respect the delivery interval to simulate real-time data flow + // If processing was faster than the interval, wait for the remaining time + if metadata.DeliveryIntervalMs > 0 && dur < time.Duration(metadata.DeliveryIntervalMs)*time.Millisecond { + slog.Debug("Waiting for delivery interval", "duration", dur, "expected", time.Duration(metadata.DeliveryIntervalMs)*time.Millisecond) + select { + case <-time.After(time.Duration(metadata.DeliveryIntervalMs)*time.Millisecond - dur): + // Wait for the remaining time + case <-ctx.Done(): + // Context was cancelled during wait + c.execution.completionChan <- ctx.Err() + return + } + } + + // Send chunk event to notify that a chunk has been completed + // This updates the collection UI and allows other components to process the chunk + chunkEvent := &events.Chunk{ExecutionId: c.execution.id, ChunkNumber: chunkIdx} + if err := c.Notify(ctx, chunkEvent); err != nil { + slog.Error("failed to notify chunk event", "error", err) + c.execution.completionChan <- fmt.Errorf("failed to notify chunk event: %w", err) + return + } + + // Update total rows processed and send status event + // This provides progress information to the collection UI + totalRowsProcessed += int64(rows) + statusEvent := &events.Status{ExecutionId: c.execution.id, RowsReceived: totalRowsProcessed, RowsEnriched: totalRowsProcessed} + if err := c.Notify(ctx, statusEvent); err != nil { + slog.Error("failed to notify status event", "error", err) + c.execution.completionChan <- fmt.Errorf("failed to notify status event: %w", err) + return + } + + chunkIdx++ // Move to next chunk + } + + // Send completion event to indicate all data has been generated + // This triggers final processing and updates the collection UI + if err := c.Notify(ctx, events.NewCompletedEvent(c.execution.id, int64(metadata.Rows), chunkIdx, nil)); err != nil { + slog.Error("failed to notify completed event", "error", err) + c.execution.completionChan <- fmt.Errorf("failed to notify completed event: %w", err) + return + } + + // Signal completion by sending nil to the completion channel + // This allows the main collection process to know that synthetic data generation is complete + c.execution.completionChan <- nil +} + +func generateSyntheticValue(column *schema.ColumnSchema, rowIndex int) any { + // Use the column's Type field directly instead of fuzzy matching on name + columnType := column.Type + + // Generate value based on exact type match (case-insensitive) + switch strings.ToUpper(columnType) { + case "VARCHAR": + return fmt.Sprintf("%s_val%d", column.ColumnName, rowIndex%100000) + case "INTEGER": + return (rowIndex % 100000) + 1 + case "DOUBLE": + return float64(rowIndex%100000) * 0.1 + case "BOOLEAN": + return rowIndex%2 == 0 + case "JSON": + return generateJSONValue(column, rowIndex) + case "TIMESTAMP": + return time.Now().AddDate(0, 0, -rowIndex%30).Format("2006-01-02 15:04:05") + default: + // Handle struct types and complex types + if strings.Contains(strings.ToUpper(columnType), "STRUCT") { + return generateStructValue(column, rowIndex) + } + // For any other unrecognized type, throw an error + panic(fmt.Sprintf("Unsupported column type '%s' for column '%s'", columnType, column.ColumnName)) + } +} + +func generateJSONValue(column *schema.ColumnSchema, rowIndex int) any { + // Generate different JSON structures based on column name + if strings.Contains(column.ColumnName, "nested_json") { + return map[string]any{ + "metadata": map[string]any{ + "created_at": time.Now().AddDate(0, 0, -rowIndex%30).Format("2006-01-02"), + "version": fmt.Sprintf("v%d.%d", rowIndex%10, rowIndex%5), + }, + "data": map[string]any{ + "field1": rowIndex % 100000, + "field2": fmt.Sprintf("field_%d", rowIndex%100000), + "field3": rowIndex%2 == 0, + }, + } + } else if strings.Contains(column.ColumnName, "array") { + return []any{ + fmt.Sprintf("item_%d", rowIndex%100000), + rowIndex % 100000, + rowIndex%2 == 0, + float64(rowIndex%100000) * 0.1, + } + } else { + // Default JSON object + return map[string]any{ + "id": rowIndex % 100000, + "name": fmt.Sprintf("item_%d", rowIndex%100000), + "value": (rowIndex % 100000) + 1, + "tags": []string{"tag1", "tag2", "tag3"}, + } + } +} + +func generateStructValue(column *schema.ColumnSchema, rowIndex int) any { + if column.StructFields == nil { + return map[string]any{ + "id": rowIndex % 100000, + "name": fmt.Sprintf("struct_%d", rowIndex%100000), + } + } + + result := make(map[string]any) + for _, field := range column.StructFields { + if field.StructFields != nil { + // Nested struct + result[field.ColumnName] = generateStructValue(field, rowIndex) + } else { + // Simple field + result[field.ColumnName] = generateSyntheticValue(field, rowIndex) + } + } + return result +} diff --git a/internal/collector/status.go b/internal/collector/status.go index f18eb910..e1601db9 100644 --- a/internal/collector/status.go +++ b/internal/collector/status.go @@ -2,6 +2,7 @@ package collector import ( "fmt" + "github.com/turbot/tailpipe/internal/database" "path/filepath" "strings" "time" @@ -9,10 +10,8 @@ import ( "github.com/dustin/go-humanize" "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/tailpipe-plugin-sdk/events" - "github.com/turbot/tailpipe-plugin-sdk/grpc/proto" "github.com/turbot/tailpipe-plugin-sdk/logging" "github.com/turbot/tailpipe-plugin-sdk/row_source" - "github.com/turbot/tailpipe/internal/parquet" ) const uiErrorsToDisplay = 15 @@ -29,19 +28,21 @@ type status struct { complete bool partitionName string fromTime *row_source.ResolvedFromTime - compactionStatus *parquet.CompactionStatus + compactionStatus *database.CompactionStatus + toTime time.Time } // Init initializes the status with the partition name and resolved from time of the collection and marks start of collection for timing -func (s *status) Init(partitionName string, fromTime *row_source.ResolvedFromTime) { +func (s *status) Init(partitionName string, fromTime *row_source.ResolvedFromTime, toTime time.Time) { s.started = time.Now() s.partitionName = partitionName s.fromTime = fromTime + s.toTime = toTime } // UpdateWithPluginStatus updates the status with the values from the plugin status event -func (s *status) UpdateWithPluginStatus(event *proto.EventStatus) { - s.Status = *events.StatusFromProto(event) +func (s *status) UpdateWithPluginStatus(event *events.Status) { + s.Status = *event } // UpdateConversionStatus updates the status with rows converted, rows the conversion failed on, and any errors @@ -55,28 +56,15 @@ func (s *status) UpdateConversionStatus(rowsConverted, failedRows int64, errors } } -// UpdateCompactionStatus updates the status with the values from the compaction status event -func (s *status) UpdateCompactionStatus(compactionStatus *parquet.CompactionStatus) { - if compactionStatus == nil { - return - } - - if s.compactionStatus == nil { - s.compactionStatus = &parquet.CompactionStatus{} - } - - s.compactionStatus.Update(*compactionStatus) -} - // CollectionHeader returns a string to display at the top of the collection status for app or alone for non-progress display func (s *status) CollectionHeader() string { // wrap the source in parentheses if it exists fromTimeSource := s.fromTime.Source if s.fromTime.Source != "" { - fromTimeSource = fmt.Sprintf("(%s)", s.fromTime.Source) + fromTimeSource = fmt.Sprintf(" (%s)", s.fromTime.Source) } - return fmt.Sprintf("\nCollecting logs for %s from %s %s\n\n", s.partitionName, s.fromTime.Time.Format(time.DateOnly), fromTimeSource) + return fmt.Sprintf("\nCollecting logs for %s from %s%s to %s\n\n", s.partitionName, s.fromTime.Time.Format(time.DateOnly), fromTimeSource, s.toTime.Format(time.DateOnly)) } // String returns a string representation of the status used as body of app display or final output for non-progress display @@ -219,14 +207,11 @@ func (s *status) displayFilesSection() string { var out strings.Builder out.WriteString("Files:\n") - if s.compactionStatus.Source == 0 && s.compactionStatus.Uncompacted == 0 { + if s.compactionStatus.InitialFiles == 0 { // no counts available, display status text out.WriteString(fmt.Sprintf(" %s\n", statusText)) } else { - // display counts source => dest - l := int64(s.compactionStatus.Source + s.compactionStatus.Uncompacted) - r := int64(s.compactionStatus.Dest + s.compactionStatus.Uncompacted) - out.WriteString(fmt.Sprintf(" Compacted: %s => %s\n", humanize.Comma(l), humanize.Comma(r))) + out.WriteString(fmt.Sprintf(" %s\n", s.compactionStatus.String())) } out.WriteString("\n") @@ -286,14 +271,22 @@ func (s *status) displayErrorsSection() string { // displayTimingSection returns a string representation of the timing section of the status (time elapsed since start of collection) func (s *status) displayTimingSection() string { duration := time.Since(s.started) - timeLabel := "Time:" // if we're complete, change the time label to show this if s.complete { - timeLabel = "Completed:" + if s.compactionStatus != nil && s.compactionStatus.Duration > 0 { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("Collection: %s\n", utils.HumanizeDuration(duration))) + sb.WriteString(fmt.Sprintf("Compaction: %s\n", utils.HumanizeDuration(s.compactionStatus.Duration))) + sb.WriteString(fmt.Sprintf("Total: %s\n", utils.HumanizeDuration(duration+s.compactionStatus.Duration))) + return sb.String() + } + return fmt.Sprintf("Completed: %s\n", utils.HumanizeDuration(duration)) + } else { + // if not complete, show elapsed time + return fmt.Sprintf("Time: %s\n", utils.HumanizeDuration(duration)) } - return fmt.Sprintf("%s %s\n", timeLabel, utils.HumanizeDuration(duration)) } // writeCountLine returns a formatted string for a count line in the status display, used for alignment and readability diff --git a/internal/collector/tui.go b/internal/collector/tui.go index 7ff1f31f..e3e5b785 100644 --- a/internal/collector/tui.go +++ b/internal/collector/tui.go @@ -1,12 +1,11 @@ package collector import ( + "github.com/turbot/tailpipe/internal/database" "strings" "time" tea "github.com/charmbracelet/bubbletea" - - "github.com/turbot/tailpipe/internal/parquet" ) type collectionModel struct { @@ -64,7 +63,7 @@ func (c collectionModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return c, nil case AwaitingCompactionMsg: // this doesn't do anything useful except trigger a view update with file compaction placeholder - cs := parquet.CompactionStatus{} + cs := database.CompactionStatus{} c.status.compactionStatus = &cs return c, nil case tickMsg: diff --git a/internal/config/connection.go b/internal/config/connection.go index e82416fa..400df1b6 100644 --- a/internal/config/connection.go +++ b/internal/config/connection.go @@ -37,12 +37,10 @@ func (c *TailpipeConnection) GetSubType() string { func (c *TailpipeConnection) ToProto() *proto.ConfigData { return &proto.ConfigData{ - //Target: c.Name(), - // TODO fix connection parsing to populate name + // Target is of form `connection.` Target: "connection." + c.Plugin, - - Hcl: c.Hcl, - Range: proto.RangeToProto(c.DeclRange), + Hcl: c.Hcl, + Range: proto.RangeToProto(c.DeclRange), } } @@ -71,47 +69,3 @@ func NewTailpipeConnection(block *hcl.Block, fullName string) (modconfig.HclReso c.UnqualifiedName = fmt.Sprintf("%s.%s", c.Plugin, c.ShortName) return c, nil } - -// TODO implement if needed https://github.com/turbot/tailpipe/issues/34 -// -//func CtyValueToConnection(value cty.Value) (_ *TailpipeConnection, err error) { -// defer func() { -// if r := recover(); r != nil { -// err = perr.BadRequestWithMessage("unable to decode connection: " + r.(string)) -// } -// }() -// -// // get the name, block type and range and use to construct a connection -// shortName := value.GetAttr("short_name").AsString() -// name := value.GetAttr("name").AsString() -// block := &hcl.Block{ -// Labels: []string{"connection", name}, -// } -// -// -// -// // now instantiate an empty connection of the correct type -// conn, err := NewTailpipeConnection(&hcl.Block{}, name) -// if err != nil { -// return nil, perr.BadRequestWithMessage("unable to decode connection: " + err.Error()) -// } -// -// // split the cty value into fields for ConnectionImpl and the derived connection, -// // (NOTE: exclude the 'env', 'type', 'resource_type' fields, which are manually added) -// baseValue, derivedValue, err := getKnownCtyFields(value, conn.GetConnectionImpl(), "env", "type", "resource_type") -// if err != nil { -// return nil, perr.BadRequestWithMessage("unable to decode connection: " + err.Error()) -// } -// // decode the base fields into the ConnectionImpl -// err = gocty.FromCtyValue(baseValue, conn.GetConnectionImpl()) -// if err != nil { -// return nil, perr.BadRequestWithMessage("unable to decode ConnectionImpl: " + err.Error()) -// } -// // decode remaining fields into the derived connection -// err = gocty.FromCtyValue(derivedValue, &conn) -// if err != nil { -// return nil, perr.BadRequestWithMessage("unable to decode connection: " + err.Error()) -// } -// -// return nil, nil -//} diff --git a/internal/config/normalize.go b/internal/config/normalize.go new file mode 100644 index 00000000..20af693c --- /dev/null +++ b/internal/config/normalize.go @@ -0,0 +1,105 @@ +package config + +import ( + "fmt" + "regexp" + "strings" +) + +// IsColumnName returns true if the string is a valid SQL column name. +// It checks that the name: +// 1. Contains only alphanumeric characters and underscores +// 2. Starts with a letter or underscore +// 3. Is not empty +// 4. Is not a DuckDB reserved keyword +func IsColumnName(s string) bool { + if s == "" { + return false + } + // Match pattern: start with letter/underscore, followed by alphanumeric/underscore + columnNameRegex := regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + if !columnNameRegex.MatchString(s) { + return false + } + + // Check for DuckDB reserved keywords + // This list is based on DuckDB's SQL dialect + reservedKeywords := map[string]bool{ + "all": true, "alter": true, "and": true, "any": true, "array": true, "as": true, + "asc": true, "between": true, "by": true, "case": true, "cast": true, "check": true, + "collate": true, "column": true, "constraint": true, "create": true, "cross": true, + "current_date": true, "current_time": true, "current_timestamp": true, "database": true, + "default": true, "delete": true, "desc": true, "distinct": true, "drop": true, + "else": true, "end": true, "except": true, "exists": true, "extract": true, + "false": true, "for": true, "foreign": true, "from": true, "full": true, + "group": true, "having": true, "in": true, "index": true, "inner": true, + "insert": true, "intersect": true, "into": true, "is": true, "join": true, + "left": true, "like": true, "limit": true, "natural": true, "not": true, + "null": true, "on": true, "or": true, "order": true, "outer": true, + "primary": true, "references": true, "right": true, "select": true, "set": true, + "some": true, "table": true, "then": true, "to": true, "true": true, + "union": true, "unique": true, "update": true, "using": true, "values": true, + "when": true, "where": true, "with": true, + } + + return !reservedKeywords[strings.ToLower(s)] +} + +// NormalizeSqlExpression processes a config value for use in SQL. +// It safely escapes and quotes strings, but passes through valid SQL expressions and column names. +// Not used fow now but may be needed if support for partition tp_index is broadened in the future to include functions +// or more complex expressions. +func NormalizeSqlExpression(expr string) string { + trimmed := strings.TrimSpace(expr) + + // NOTE: for now we only + + // Case 1: already quoted SQL string literal + if strings.HasPrefix(trimmed, "'") && strings.HasSuffix(trimmed, "'") && len(trimmed) >= 2 { + inner := trimmed[1 : len(trimmed)-1] + escaped := strings.ReplaceAll(inner, "'", "''") + return fmt.Sprintf("'%s'", escaped) + } + + // Case 2: looks like SQL expression + if looksLikeSQLExpression(trimmed) { + return trimmed + } + + // Case 3: bare identifier (column name), e.g., timestamp, event_id + if isBareIdentifier(trimmed) { + return trimmed + } + + // Case 4: fallback ā€” treat as string literal + escaped := strings.ReplaceAll(trimmed, "'", "''") + return fmt.Sprintf("'%s'", escaped) +} + +func looksLikeSQLExpression(s string) bool { + s = strings.ToLower(strings.TrimSpace(s)) + + // Heuristics: contains operators or function-style tokens + if strings.ContainsAny(s, "()*/%+-=<>|") || strings.Contains(s, "::") { + return true + } + + // Known SQL keywords or functions + sqlExprPattern := regexp.MustCompile(`(?i)\b(select|case|when|then|else|end|cast|concat|coalesce|nullif|date_trunc|extract)\b`) + if sqlExprPattern.MatchString(s) { + return true + } + + // If it contains multiple words (e.g., space), and doesn't match other rules, treat it as not an expression + if strings.Contains(s, " ") { + return false + } + + return false +} + +// isBareIdentifier returns true if s is a simple SQL identifier (e.g., a column name). +func isBareIdentifier(s string) bool { + identifierPattern := regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + return identifierPattern.MatchString(s) +} diff --git a/internal/config/normalize_test.go b/internal/config/normalize_test.go new file mode 100644 index 00000000..e09dc5e9 --- /dev/null +++ b/internal/config/normalize_test.go @@ -0,0 +1,269 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_normalizeExpression(t *testing.T) { + type args struct { + expr string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "already quoted string literal", + args: args{ + expr: "'hello world'", + }, + want: "'hello world'", + }, + { + name: "quoted string with single quote", + args: args{ + expr: "'hello'world'", + }, + want: "'hello''world'", + }, + { + name: "SQL expression with operators", + args: args{ + expr: "column1 + column2", + }, + want: "column1 + column2", + }, + { + name: "SQL expression with function", + args: args{ + expr: "COALESCE(column1, 'default')", + }, + want: "COALESCE(column1, 'default')", + }, + { + name: "bare identifier", + args: args{ + expr: "event_id", + }, + want: "event_id", + }, + { + name: "bare identifier with underscore", + args: args{ + expr: "user_name_123", + }, + want: "user_name_123", + }, + { + name: "string literal needs quoting", + args: args{ + expr: "hello world", + }, + want: "'hello world'", + }, + { + name: "string literal with single quote needs escaping", + args: args{ + expr: "hello'world", + }, + want: "'hello''world'", + }, + { + name: "empty string", + args: args{ + expr: "", + }, + want: "''", + }, + { + name: "whitespace only", + args: args{ + expr: " ", + }, + want: "''", + }, + { + name: "complex SQL expression with multiple operators", + args: args{ + expr: "(column1 + column2) * (column3 - column4)", + }, + want: "(column1 + column2) * (column3 - column4)", + }, + { + name: "SQL expression with type cast", + args: args{ + expr: "column1::timestamp", + }, + want: "column1::timestamp", + }, + { + name: "SQL expression with CASE statement", + args: args{ + expr: "CASE WHEN column1 > 0 THEN 'positive' ELSE 'negative' END", + }, + want: "CASE WHEN column1 > 0 THEN 'positive' ELSE 'negative' END", + }, + { + name: "string with special characters", + args: args{ + expr: "hello\nworld\twith\rspecial chars", + }, + want: "'hello\nworld\twith\rspecial chars'", + }, + { + name: "string with unicode characters", + args: args{ + expr: "hello äø–ē•Œ", + }, + want: "'hello äø–ē•Œ'", + }, + { + name: "SQL expression with date function", + args: args{ + expr: "date_trunc('day', timestamp_column)", + }, + want: "date_trunc('day', timestamp_column)", + }, + { + name: "SQL expression with multiple functions", + args: args{ + expr: "COALESCE(NULLIF(column1, ''), 'default')", + }, + want: "COALESCE(NULLIF(column1, ''), 'default')", + }, + { + name: "identifier starting with underscore", + args: args{ + expr: "_private_column", + }, + want: "_private_column", + }, + { + name: "identifier with numbers", + args: args{ + expr: "column_123", + }, + want: "column_123", + }, + { + name: "string with multiple single quotes", + args: args{ + expr: "O'Reilly's book", + }, + want: "'O''Reilly''s book'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NormalizeSqlExpression(tt.args.expr) + + assert.Equalf(t, tt.want, got, "NormalizeSqlExpression(%v)", tt.args.expr) + }) + } +} + +func TestIsColumnName(t *testing.T) { + type args struct { + s string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "valid simple column name", + args: args{ + s: "column1", + }, + want: true, + }, + { + name: "valid column name with underscore", + args: args{ + s: "user_name", + }, + want: true, + }, + { + name: "valid column name starting with underscore", + args: args{ + s: "_private_column", + }, + want: true, + }, + { + name: "valid column name with numbers", + args: args{ + s: "column_123", + }, + want: true, + }, + { + name: "empty string", + args: args{ + s: "", + }, + want: false, + }, + { + name: "starts with number", + args: args{ + s: "1column", + }, + want: false, + }, + { + name: "contains special characters", + args: args{ + s: "column@name", + }, + want: false, + }, + { + name: "contains spaces", + args: args{ + s: "column name", + }, + want: false, + }, + { + name: "reserved keyword - select", + args: args{ + s: "select", + }, + want: false, + }, + { + name: "reserved keyword - from", + args: args{ + s: "from", + }, + want: false, + }, + { + name: "reserved keyword - where", + args: args{ + s: "where", + }, + want: false, + }, + { + name: "reserved keyword - case insensitive", + args: args{ + s: "SELECT", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsColumnName(tt.args.s); got != tt.want { + t.Errorf("IsColumnName() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/config/partition.go b/internal/config/partition.go index 4b000e6e..8086f82f 100644 --- a/internal/config/partition.go +++ b/internal/config/partition.go @@ -21,6 +21,13 @@ func init() { registerResourceWithSubType(schema.BlockTypePartition) } +type SyntheticMetadata struct { + Columns int + Rows int + ChunkSize int + DeliveryIntervalMs int +} + type Partition struct { modconfig.HclResourceImpl // required to allow partial decoding @@ -43,7 +50,12 @@ type Partition struct { // the config location ConfigRange hclhelpers.Range `cty:"config_range"` // an option filter in the format of a SQL where clause - Filter string + Filter string `cty:"filter"` + // the sql column to use for the tp_index + TpIndexColumn string `cty:"tp_index"` + + // if this is a synthetic partition for testing, this will be non-null + SyntheticMetadata *SyntheticMetadata } func NewPartition(block *hcl.Block, fullName string) (modconfig.HclResource, hcl.Diagnostics) { @@ -65,69 +77,80 @@ func NewPartition(block *hcl.Block, fullName string) (modconfig.HclResource, hcl return c, nil } -func (c *Partition) SetConfigHcl(u *HclBytes) { +func (p *Partition) SetConfigHcl(u *HclBytes) { if u == nil { return } - c.Config = u.Hcl - c.ConfigRange = u.Range + p.Config = u.Hcl + p.ConfigRange = u.Range } -func (c *Partition) InferPluginName(v *versionfile.PluginVersionFile) string { +func (p *Partition) InferPluginName(v *versionfile.PluginVersionFile) string { // NOTE: we cannot call the TailpipeConfig.GetPluginForTable function as tailpipe config is not populated yet - if c.CustomTable != nil { - return constants.CorePluginName + if p.CustomTable != nil { + return constants.CorePluginInstallStream() } - return GetPluginForTable(c.TableName, v.Plugins) + return GetPluginForTable(p.TableName, v.Plugins) } -func (c *Partition) AddFilter(filter string) { - if c.Filter == "" { - c.Filter = filter +func (p *Partition) AddFilter(filter string) { + if p.Filter == "" { + p.Filter = filter } else { - c.Filter += " and " + filter + p.Filter += " and " + filter } } -func (c *Partition) CollectionStatePath(collectionDir string) string { +func (p *Partition) CollectionStatePath(collectionDir string) string { // return the path to the collection state file - return filepath.Join(collectionDir, fmt.Sprintf("collection_state_%s_%s.json", c.TableName, c.ShortName)) + return filepath.Join(collectionDir, fmt.Sprintf("collection_state_%s_%s.json", p.TableName, p.ShortName)) } -func (c *Partition) Validate() hcl.Diagnostics { +func (p *Partition) Validate() hcl.Diagnostics { var diags hcl.Diagnostics + // validate source block is present + if p.Source.Type == "" { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Partition '%s' is missing required source block", p.GetUnqualifiedName()), + Subject: p.ConfigRange.HclRange().Ptr(), + }) + } + // validate filter - if c.Filter != "" { - diags = append(diags, c.validateFilter()...) + if p.Filter != "" { + diags = append(diags, p.validateFilter()...) } + moreDiags := p.validateIndexExpression() + diags = append(diags, moreDiags...) return diags } // CtyValue implements CtyValueProvider // (note this must be implemented by each resource, we cannot rely on the HclResourceImpl implementation as it will // only serialise its own properties) ) -func (c *Partition) CtyValue() (cty.Value, error) { - return cty_helpers.GetCtyValue(c) +func (p *Partition) CtyValue() (cty.Value, error) { + return cty_helpers.GetCtyValue(p) } -func (c *Partition) validateFilter() hcl.Diagnostics { +func (p *Partition) validateFilter() hcl.Diagnostics { var diags hcl.Diagnostics // check for `;` to prevent multiple statements - if strings.Contains(c.Filter, ";") { + if strings.Contains(p.Filter, ";") { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, - Summary: fmt.Sprintf("Partition %s contains invalid filter", c.GetUnqualifiedName()), + Summary: fmt.Sprintf("Partition %s contains invalid filter", p.GetUnqualifiedName()), Detail: "multiple expressions are not supported in partition filters, should not contain ';'.", }) } // check for `/*`, `*/`, `--` to prevent comments - if strings.Contains(c.Filter, "/*") || strings.Contains(c.Filter, "*/") || strings.Contains(c.Filter, "--") { + if strings.Contains(p.Filter, "/*") || strings.Contains(p.Filter, "*/") || strings.Contains(p.Filter, "--") { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, - Summary: fmt.Sprintf("Partition %s contains invalid filter", c.GetUnqualifiedName()), + Summary: fmt.Sprintf("Partition %s contains invalid filter", p.GetUnqualifiedName()), Detail: "comments are not supported in partition filters, should not contain comment identifiers '/*', '*/' or '--'.", }) } @@ -147,13 +170,13 @@ func (c *Partition) validateFilter() hcl.Diagnostics { "with ", } - lower := strings.ToLower(c.Filter) + lower := strings.ToLower(p.Filter) for _, s := range forbiddenStrings { if strings.Contains(lower, s) { str := strings.Trim(s, " ") diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, - Summary: fmt.Sprintf("Partition %s contains invalid filter", c.GetUnqualifiedName()), + Summary: fmt.Sprintf("Partition %s contains invalid filter", p.GetUnqualifiedName()), Detail: fmt.Sprintf("should not contain keyword '%s' in filter, unless used as a quoted identifier ('\"%s\"') to prevent unintended behavior.", str, str), }) } @@ -162,18 +185,64 @@ func (c *Partition) validateFilter() hcl.Diagnostics { return diags } +func (p *Partition) validateIndexExpression() hcl.Diagnostics { + var diags hcl.Diagnostics + + if p.TpIndexColumn == "" { + p.TpIndexColumn = "'default'" + return diags + } + + // check for `;` to prevent multiple statements + if strings.Contains(p.Filter, ";") { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Partition %s contains invalid filter", p.GetUnqualifiedName()), + Detail: "multiple expressions are not supported in partition filters, should not contain ';'.", + Subject: p.GetDeclRange(), + }) + } + // check for `/*`, `*/`, `--` to prevent comments + if strings.Contains(p.Filter, "/*") || strings.Contains(p.Filter, "*/") || strings.Contains(p.Filter, "--") { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Partition %s contains invalid filter", p.GetUnqualifiedName()), + Detail: "comments are not supported in partition filters, should not contain comment identifiers '/*', '*/' or '--'.", + Subject: p.GetDeclRange(), + }) + } + if diags.HasErrors() { + return diags + } + + // tp_index must be a column name - validate it + if !IsColumnName(p.TpIndexColumn) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Partition %s has an invalid tp_index expression", p.GetUnqualifiedName()), + Detail: fmt.Sprintf("tp_index '%s' is not a valid column name. It should be a simple column name without any SQL expressions or functions.", p.TpIndexColumn), + Subject: p.GetDeclRange(), + }) + return diags + } + // wrap in double quotes + p.TpIndexColumn = fmt.Sprintf(`"%s"`, p.TpIndexColumn) + + return diags +} + // GetFormat returns the format for this partition, if either the source or the custom table has one -func (c *Partition) GetFormat() *Format { - var format = c.Source.Format - if format == nil && c.CustomTable != nil { +func (p *Partition) GetFormat() *Format { + var format = p.Source.Format + if format == nil && p.CustomTable != nil { // if the source does not provide a format, use the custom table format - format = c.CustomTable.DefaultSourceFormat + format = p.CustomTable.DefaultSourceFormat } return format } -func (c *Partition) FormatSupportsDirectConversion() bool { - format := c.GetFormat() +func (p *Partition) FormatSupportsDirectConversion() bool { + format := p.GetFormat() if format == nil { return false } diff --git a/internal/config/source.go b/internal/config/source.go index 3efcf3c0..b5ab318c 100644 --- a/internal/config/source.go +++ b/internal/config/source.go @@ -1,6 +1,7 @@ package config import ( + "github.com/turbot/pipe-fittings/v2/hclhelpers" "github.com/turbot/tailpipe-plugin-sdk/grpc/proto" ) @@ -13,10 +14,23 @@ type Source struct { Config *HclBytes `cty:"config"` } +func NewSource(sourceType string) *Source { + return &Source{ + Type: sourceType, + Config: &HclBytes{ + Hcl: []byte{}, + Range: hclhelpers.Range{}, + }, + } +} func (s *Source) ToProto() *proto.ConfigData { + var hcl []byte + if s.Config != nil { + hcl = s.Config.Hcl + } return &proto.ConfigData{ Target: "source." + s.Type, - Hcl: s.Config.Hcl, + Hcl: hcl, Range: proto.RangeToProto(s.Config.Range.HclRange()), } } diff --git a/internal/config/table.go b/internal/config/table.go index e990fd10..154967ef 100644 --- a/internal/config/table.go +++ b/internal/config/table.go @@ -9,6 +9,7 @@ import ( "github.com/turbot/pipe-fittings/v2/cty_helpers" "github.com/turbot/pipe-fittings/v2/hclhelpers" "github.com/turbot/pipe-fittings/v2/modconfig" + "github.com/turbot/tailpipe-plugin-sdk/constants" "github.com/turbot/tailpipe-plugin-sdk/grpc/proto" "github.com/turbot/tailpipe-plugin-sdk/schema" "github.com/zclconf/go-cty/cty" @@ -20,12 +21,13 @@ type Table struct { // required to allow partial decoding Remain hcl.Body `hcl:",remain" json:"-"` - // the default format for this table (todo make a map keyed by source name?) + // the default format for this table DefaultSourceFormat *Format `hcl:"format" cty:"format"` Columns []Column `hcl:"column,block" cty:"columns"` // should we include ALL source fields in addition to any defined columns, or ONLY include the columns defined + // default to automap ALL source fields (*) MapFields []string `hcl:"map_fields,optional" cty:"map_fields"` // the default null value for the table (may be overridden for specific columns) NullIf string `hcl:"null_if,optional" cty:"null_if"` @@ -78,6 +80,15 @@ func (t *Table) Validate() hcl.Diagnostics { var validationErrors []string for _, col := range t.Columns { + // if the table definition contains a mapping for tp_index - return a warning that this will be ignored + if col.Name == constants.TpIndex && (typehelpers.SafeString(col.Source) != "" || typehelpers.SafeString(col.Transform) != "") { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: fmt.Sprintf("Table '%s' contains a mapping for column 'tp_index' which will be ignored. To set the source column mapping for tp_index, set the 'tp_index' property of the partition config", t.ShortName), + Subject: t.DeclRange.Ptr(), + }) + continue + } // if the column is options, a type must be specified // - this is to ensure we can determine the column type in the case of the column being missing in the source data // (if the column is required, the column being missing would cause an error so this problem will not arise) @@ -112,5 +123,6 @@ func (t *Table) Validate() hcl.Diagnostics { Subject: t.DeclRange.Ptr(), }) } + // return diags } diff --git a/internal/config/tailpipe_config.go b/internal/config/tailpipe_config.go index 9a0e5970..b63e5899 100644 --- a/internal/config/tailpipe_config.go +++ b/internal/config/tailpipe_config.go @@ -5,8 +5,11 @@ import ( "strings" "github.com/hashicorp/hcl/v2" + "github.com/spf13/viper" + "github.com/turbot/pipe-fittings/v2/constants" "github.com/turbot/pipe-fittings/v2/modconfig" "github.com/turbot/pipe-fittings/v2/plugin" + "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/pipe-fittings/v2/versionfile" ) @@ -33,6 +36,10 @@ func NewTailpipeConfig() *TailpipeConfig { func (c *TailpipeConfig) Add(resource modconfig.HclResource) error { switch t := resource.(type) { case *Partition: + // Check if a partition with the same name already exists + if _, exists := c.Partitions[t.GetUnqualifiedName()]; exists { + return fmt.Errorf("partition %s already exists for table %s", t.ShortName, t.TableName) + } c.Partitions[t.GetUnqualifiedName()] = t return nil case *TailpipeConnection: @@ -72,6 +79,10 @@ func (c *TailpipeConfig) InitPartitions(versionMap *versionfile.PluginVersionFil // if the plugin is not set, infer it from the table if partition.Plugin == nil { partition.Plugin = plugin.NewPlugin(partition.InferPluginName(versionMap)) + // set memory limit on plugin struct + if viper.IsSet(constants.ArgMemoryMaxMbPlugin) { + partition.Plugin.MemoryMaxMb = utils.ToPointer(viper.GetInt(constants.ArgMemoryMaxMbPlugin)) + } } } } diff --git a/internal/constants/connect.go b/internal/constants/connect.go new file mode 100644 index 00000000..c91b9948 --- /dev/null +++ b/internal/constants/connect.go @@ -0,0 +1,6 @@ +package constants + +import "time" + +// InitFileMaxAge is the maximum age of an db init file before it is cleaned up +const InitFileMaxAge = 24 * time.Hour diff --git a/internal/constants/database.go b/internal/constants/database.go deleted file mode 100644 index f7667e5f..00000000 --- a/internal/constants/database.go +++ /dev/null @@ -1,8 +0,0 @@ -package constants - -import "time" - -const ( - TailpipeDbName = "tailpipe.db" - DbFileMaxAge = 24 * time.Hour -) diff --git a/internal/constants/duckdb_extensions.go b/internal/constants/duckdb_extensions.go deleted file mode 100644 index e7d02979..00000000 --- a/internal/constants/duckdb_extensions.go +++ /dev/null @@ -1,3 +0,0 @@ -package constants - -var DuckDbExtensions = []string{"json", "inet"} diff --git a/internal/constants/metaquery_commands.go b/internal/constants/metaquery_commands.go index f33f81ce..978e51ad 100644 --- a/internal/constants/metaquery_commands.go +++ b/internal/constants/metaquery_commands.go @@ -3,9 +3,7 @@ package constants // Metaquery commands const ( - //CmdTableList = ".tables" // List all tables - CmdOutput = ".output" // Set output mode - //CmdTiming = ".timing" // Toggle query timer + CmdOutput = ".output" // Set output mode CmdHeaders = ".header" // Toggle headers output CmdSeparator = ".separator" // Set the column separator CmdExit = ".exit" // Exit the interactive prompt diff --git a/internal/constants/plugin.go b/internal/constants/plugin.go index cabc2b7c..fc44e390 100644 --- a/internal/constants/plugin.go +++ b/internal/constants/plugin.go @@ -1,11 +1,61 @@ package constants +import ( + "strings" +) + const ( - CorePluginName = "core" - CorePluginFullName = "hub.tailpipe.io/plugins/turbot/core@latest" - MinCorePluginVersion = "v0.2.0" + + // MinCorePluginVersion should be set for production releases - it is the minimum version of the core plugin that is required + MinCorePluginVersion = "v0.2.10" + // CorePluginVersion may be set for pre-release versions - it allows us to pin a pre-release version of the core plugin + // NOTE: they must NOT both be set + CorePluginVersion = "" // TailpipeHubOCIBase is the tailpipe hub URL TailpipeHubOCIBase = "hub.tailpipe.io/" + // BaseImageRef is the prefix for all tailpipe plugin images BaseImageRef = "ghcr.io/turbot/tailpipe" ) + +// CorePluginRequiredVersionConstraint returns a version constraint for the required core plugin version +// normally we set the core version by setting constants.MinCorePluginVersion +// However if we want ot pin to a specific version (e.g. an rc version) we can set constants.CorePluginVersion instead +// one of constants.CorePluginVersion and constants.MinCorePluginVersion may be set +// if both are set this is a bug +func CorePluginRequiredVersionConstraint() (requiredConstraint string) { + if CorePluginVersion == "" && MinCorePluginVersion == "" { + panic("one of constants.CorePluginName or constants.MinCorePluginVersion must be set") + } + if CorePluginVersion != "" && MinCorePluginVersion != "" { + panic("both constants.CorePluginVersion and constants.MinCorePluginVersion are set, this is a bug") + } + if MinCorePluginVersion != "" { + requiredConstraint = ">=" + MinCorePluginVersion + return requiredConstraint + } + + // so CorePluginVersion is set - return as-is + return CorePluginVersion +} + +// CorePluginInstallStream returns the plugin stream used to install the core plugin +// under normal circumstances (i.e. if MinCorePluginVersion is set) this is "core@latest" +func CorePluginInstallStream() string { + var installConstraint string + if MinCorePluginVersion != "" { + installConstraint = "latest" + } else { + // so CorePluginVersion is set + // tactical - trim 'v' as installation expects no v + installConstraint = strings.TrimPrefix(CorePluginVersion, "v") + + } + + return "core@" + installConstraint +} + +func CorePluginFullName() string { + installStream := CorePluginInstallStream() + return "hub.tailpipe.io/plugins/turbot/" + installStream +} diff --git a/internal/database/backup.go b/internal/database/backup.go new file mode 100644 index 00000000..05d106c4 --- /dev/null +++ b/internal/database/backup.go @@ -0,0 +1,138 @@ +package database + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + "time" + + "github.com/turbot/pipe-fittings/v2/utils" + "github.com/turbot/tailpipe/internal/config" +) + +// BackupDucklakeMetadata creates a timestamped backup of the DuckLake metadata database. +// It creates backup files with format: metadata.sqlite.backup.YYYYMMDDHHMMSS +// and also backs up the WAL file if it exists: +// - metadata.sqlite-wal.backup.YYYYMMDDHHMMSS +// It removes any existing backup files to maintain only the most recent backup. +// +// The backup is created in the same directory as the original database file. +// If the database file doesn't exist, no backup is created and no error is returned. +// +// Returns an error if the backup operation fails. +func BackupDucklakeMetadata() error { + // Get the path to the DuckLake metadata database + dbPath := config.GlobalWorkspaceProfile.GetDucklakeDbPath() + + // Check if the database file exists + if _, err := os.Stat(dbPath); os.IsNotExist(err) { + slog.Debug("DuckLake metadata database does not exist, skipping backup", "path", dbPath) + return nil + } else if err != nil { + return fmt.Errorf("failed to check if database exists: %w", err) + } + + // Generate timestamp for backup filename + timestamp := time.Now().Format("20060102150405") // YYYYMMDDHHMMSS format + + // Create backup filenames + dbDir := filepath.Dir(dbPath) + mainBackupFilename := fmt.Sprintf("metadata.sqlite.backup.%s", timestamp) + mainBackupPath := filepath.Join(dbDir, mainBackupFilename) + + // Also prepare paths for WAL file + walPath := dbPath + "-wal" + walBackupFilename := fmt.Sprintf("metadata.sqlite-wal.backup.%s", timestamp) + walBackupPath := filepath.Join(dbDir, walBackupFilename) + + slog.Info("Creating backup of DuckLake metadata database", "source", dbPath, "backup", mainBackupPath) + + // Create the main database backup first + if err := utils.CopyFile(dbPath, mainBackupPath); err != nil { + return fmt.Errorf("failed to create main database backup: %w", err) + } + + // Backup WAL file if it exists + if _, err := os.Stat(walPath); err == nil { + if err := utils.CopyFile(walPath, walBackupPath); err != nil { + slog.Warn("Failed to backup WAL file", "source", walPath, "error", err) + // Continue - WAL backup failure is not critical + } else { + slog.Debug("Successfully backed up WAL file", "backup", walBackupPath) + } + } + + slog.Info("Successfully created backup of DuckLake metadata database", "backup", mainBackupPath) + + // Clean up old backup files after successfully creating the new one + if err := cleanupOldBackups(dbDir, timestamp); err != nil { + slog.Warn("Failed to clean up old backup files", "error", err) + // Don't return error - the backup was successful, cleanup is just housekeeping + } + return nil +} + +// isBackupFile checks if a filename matches any of the backup patterns +func isBackupFile(filename string) bool { + backupPrefixes := []string{ + "metadata.sqlite.backup.", + "metadata.sqlite-wal.backup.", + } + + for _, prefix := range backupPrefixes { + if strings.HasPrefix(filename, prefix) { + return true + } + } + return false +} + +// shouldRemoveBackup determines if a backup file should be removed +func shouldRemoveBackup(filename, excludeTimestamp string) bool { + if !isBackupFile(filename) { + return false + } + // Don't remove files with the current timestamp + return !strings.HasSuffix(filename, "."+excludeTimestamp) +} + +// cleanupOldBackups removes all existing backup files in the specified directory, +// except for the newly created backup files with the given timestamp. +// Backup files are identified by the patterns: +// - metadata.sqlite.backup.* +// - metadata.sqlite-wal.backup.* +func cleanupOldBackups(dir, excludeTimestamp string) error { + entries, err := os.ReadDir(dir) + if err != nil { + return fmt.Errorf("failed to read directory: %w", err) + } + + var deletedCount int + for _, entry := range entries { + if entry.IsDir() { + continue + } + + filename := entry.Name() + if !shouldRemoveBackup(filename, excludeTimestamp) { + continue + } + + backupPath := filepath.Join(dir, filename) + if err := os.Remove(backupPath); err != nil { + slog.Warn("Failed to remove old backup file", "file", backupPath, "error", err) + // Continue removing other files even if one fails + } else { + slog.Debug("Removed old backup file", "file", backupPath) + deletedCount++ + } + } + + if deletedCount > 0 { + slog.Debug("Cleaned up old backup files", "count", deletedCount) + } + + return nil +} diff --git a/internal/database/cleanup.go b/internal/database/cleanup.go new file mode 100644 index 00000000..5a189362 --- /dev/null +++ b/internal/database/cleanup.go @@ -0,0 +1,136 @@ +package database + +import ( + "context" + "fmt" + "log/slog" + "time" + + "github.com/turbot/pipe-fittings/v2/constants" + "github.com/turbot/tailpipe/internal/config" +) + +// DeletePartition deletes data for the specified partition and date range from the given Ducklake connected database. +func DeletePartition(ctx context.Context, partition *config.Partition, from, to time.Time, db *DuckDb) (rowCount int, err error) { + // First check if the table exists using DuckLake metadata + tableExistsQuery := fmt.Sprintf(`select exists (select 1 from %s.ducklake_table where table_name = ?)`, constants.DuckLakeMetadataCatalog) + var tableExists bool + if err := db.QueryRowContext(ctx, tableExistsQuery, partition.TableName).Scan(&tableExists); err != nil { + return 0, fmt.Errorf("failed to check if table exists: %w", err) + } + + if !tableExists { + // Table doesn't exist, return 0 rows affected (not an error) + return 0, nil + } + + // build a delete query for the partition + // Note: table names cannot be parameterized, so we use string formatting for the table name + query := fmt.Sprintf(`delete from "%s" where tp_partition = ? and tp_timestamp >= ? and tp_timestamp <= ?`, partition.TableName) + // Execute the query with parameters for the partition and date range + result, err := db.ExecContext(ctx, query, partition.ShortName, from, to) + if err != nil { + return 0, fmt.Errorf("failed to delete partition: %w", err) + } + + // Get the number of rows affected by the delete operation + rowsAffected, err := result.RowsAffected() + if err != nil { + return 0, fmt.Errorf("failed to get rows affected count: %w", err) + } + rowCount = int(rowsAffected) + + // Only perform cleanup if we actually deleted some rows + if rowCount > 0 { + if err = DucklakeCleanup(ctx, db); err != nil { + return 0, err + } + } + + return rowCount, nil +} + +// DucklakeCleanup performs removes old snapshots deletes expired and unused parquet files from the DuckDB database. +func DucklakeCleanup(ctx context.Context, db *DuckDb) error { + slog.Info("Cleaning up DuckLake snapshots and expired files") + // now clean old snapshots + if err := expirePrevSnapshots(ctx, db); err != nil { + return err + } + // delete expired files + if err := cleanupExpiredFiles(ctx, db); err != nil { + return err + } + return nil +} + +// expirePrevSnapshots expires all snapshots but the latest +// Ducklake stores a snapshot corresponding to each database operation - this allows the tracking of the history of changes +// However we do not need (currently) take advantage of this ducklake functionality, so we can remove all but the latest snapshot +// To do this we get the date of the most recent snapshot and then expire all snapshots older than that date. +// We then call ducklake_cleanup to remove the expired files. +func expirePrevSnapshots(ctx context.Context, db *DuckDb) error { + slog.Info("Expiring old DuckLake snapshots") + defer slog.Info("DuckLake snapshot expiration complete") + + // 1) get the timestamp of the latest snapshot from the metadata schema + var latestTimestamp string + query := fmt.Sprintf(`select snapshot_time from %s.ducklake_snapshot order by snapshot_id desc limit 1`, constants.DuckLakeMetadataCatalog) + + err := db.QueryRowContext(ctx, query).Scan(&latestTimestamp) + if err != nil { + return fmt.Errorf("failed to get latest snapshot timestamp: %w", err) + } + + // Parse the snapshot time + // NOTE: rather than cast as timestamp, we read as a string then remove any timezone component + // This is because of the dubious behaviour of ducklake_expire_snapshots described below + // try various formats + formats := []string{ + "2006-01-02 15:04:05.999-07:00", // +05:30 + "2006-01-02 15:04:05.999-07", // +01 + "2006-01-02 15:04:05.999", // no timezone + } + var parsedTime time.Time + for _, format := range formats { + parsedTime, err = time.Parse(format, latestTimestamp) + if err == nil { + break + } + } + if err != nil { + return fmt.Errorf("failed to parse snapshot time '%s': %w", latestTimestamp, err) + } + + // format the time + // Note: ducklake_expire_snapshots expects a local time without timezone, + // i.e if the time is '2025-08-26 13:25:10.365 +0100', we should pass '2025-08-26 13:25:10.365' + formattedTime := parsedTime.Format("2006-01-02 15:04:05.000") + slog.Debug("Latest snapshot timestamp", "timestamp", latestTimestamp) + + // 2) expire all snapshots older than the latest one + // Note: ducklake_expire_snapshots uses named parameters which cannot be parameterized with standard SQL placeholders + expireQuery := fmt.Sprintf(`call ducklake_expire_snapshots('%s', older_than => '%s')`, constants.DuckLakeCatalog, formattedTime) + + _, err = db.ExecContext(ctx, expireQuery) + if err != nil { + return fmt.Errorf("failed to expire old snapshots: %w", err) + } + + return nil +} + +// cleanupExpiredFiles deletes and files marked as expired in the ducklake system. +func cleanupExpiredFiles(ctx context.Context, db *DuckDb) error { + slog.Info("Cleaning up expired files in DuckLake") + defer slog.Info("DuckLake expired files cleanup complete") + + cleanupQuery := fmt.Sprintf("call ducklake_cleanup_old_files('%s', cleanup_all => true)", constants.DuckLakeCatalog) + + _, err := db.ExecContext(ctx, cleanupQuery) + if err != nil { + return fmt.Errorf("failed to cleanup expired files: %w", err) + } + + return nil +} diff --git a/internal/database/compact.go b/internal/database/compact.go new file mode 100644 index 00000000..c81b5a0a --- /dev/null +++ b/internal/database/compact.go @@ -0,0 +1,435 @@ +package database + +import ( + "context" + "database/sql" + "fmt" + + "log/slog" + "strings" + "time" + + "github.com/turbot/pipe-fittings/v2/backend" + "github.com/turbot/pipe-fittings/v2/constants" +) + +const ( + // maxCompactionRowsPerChunk is the maximum number of rows to compact in a single insert operation + maxCompactionRowsPerChunk = 5_000_000 +) + +func CompactDataFiles(ctx context.Context, db *DuckDb, updateFunc func(CompactionStatus), reindex bool, patterns ...*PartitionPattern) error { + slog.Info("Compacting DuckLake data files") + + t := time.Now() + + // get a list of partition key combinations which match any of the patterns + partitionKeys, err := getPartitionKeysMatchingPattern(ctx, db, patterns) + if err != nil { + return fmt.Errorf("failed to get partition keys requiring compaction: %w", err) + } + + if len(partitionKeys) == 0 { + slog.Info("No matching partitions found for compaction") + return nil + } + + status, err := orderDataFiles(ctx, db, updateFunc, partitionKeys, reindex) + if err != nil { + slog.Error("Failed to compact DuckLake parquet files", "error", err) + return err + } + + // now expire unused snapshots + if err := expirePrevSnapshots(ctx, db); err != nil { + slog.Error("Failed to expire previous DuckLake snapshots", "error", err) + return err + } + + // so we should now have multiple, time ordered parquet files + // now merge the the parquet files in the duckdb database + // the will minimise the parquet file count to the optimum + if err := mergeParquetFiles(ctx, db); err != nil { + slog.Error("Failed to merge DuckLake parquet files", "error", err) + return err + } + + // delete unused files + if err := cleanupExpiredFiles(ctx, db); err != nil { + slog.Error("Failed to cleanup expired files", "error", err) + return err + } + + // get the file count after merging and cleanup + err = status.getFinalFileCounts(ctx, db, partitionKeys) + if err != nil { + // just log + slog.Error("Failed to get final file counts", "error", err) + } + // set the compaction time + status.Duration = time.Since(t) + + // call final update + updateFunc(*status) + + slog.Info("DuckLake compaction complete", "source_file_count", status.InitialFiles, "destination_file_count", status.FinalFiles) + return nil +} + +// mergeParquetFiles combines adjacent parquet files in the DuckDB database. +func mergeParquetFiles(ctx context.Context, db *DuckDb) error { + if _, err := db.ExecContext(ctx, "call merge_adjacent_files()"); err != nil { + if ctx.Err() != nil { + return err + } + return fmt.Errorf("failed to merge parquet files: %w", err) + } + return nil +} + +// we order data files as follows: +// - get list of partition keys matching patterns. For each key: +// - analyze file fragmentation to identify overlapping time ranges +// - for each overlapping time range, reorder all data in that range +// - delete original unordered entries for that time range +func orderDataFiles(ctx context.Context, db *DuckDb, updateFunc func(CompactionStatus), partitionKeys []*partitionKey, reindex bool) (*CompactionStatus, error) { + slog.Info("Ordering DuckLake data files") + + status := NewCompactionStatus() + // get total file and row count into status + err := status.getInitialCounts(ctx, db, partitionKeys) + if err != nil { + return nil, err + } + + // map of table columns, allowing us to lazy load them + tableColumnLookup := make(map[string][]string) + + // build list of partition keys to reorder + var reorderList []*reorderMetadata + + status.Message = "identifying files to reorder" + updateFunc(*status) + + // Process each partition key to determine if we need to reorder + for _, pk := range partitionKeys { + // determine which files are not time ordered and build a set of time ranges which need reordering + // (NOTS: if we are reindexing, we need to reorder the ALL data for the partition key) + reorderMetadata, err := getTimeRangesToReorder(ctx, db, pk, reindex) + if err != nil { + slog.Error("failed to get unorderedRanges", "partition", pk, "error", err) + return nil, err + } + + // if no files out of order, nothing to do + if reorderMetadata != nil { + reorderList = append(reorderList, reorderMetadata) + } else { + slog.Debug("Partition key is not out of order - skipping reordering", + "tp_table", pk.tpTable, + "tp_partition", pk.tpPartition, + // "tp_index", pk.tpIndex, + "year", pk.year, + "month", pk.month, + ) + } + } + + // now get the total rows to reorder + for _, rm := range reorderList { + status.InitialFiles += rm.pk.fileCount + status.RowsToCompact += rm.rowCount + } + + // clear message - it will be sent on next update func + status.Message = "" + + // now iterate over reorderlist to do reordering + for _, rm := range reorderList { + pk := rm.pk + + // get the columns for this table - check map first - if not present, read from metadata and populate the map + columns, err := getColumns(ctx, db, pk.tpTable, tableColumnLookup) + if err != nil { + slog.Error("failed to get columns", "table", pk.tpTable, "error", err) + return nil, err + } + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + // This is a system failure - stop everything + return nil, fmt.Errorf("failed to begin transaction for partition %v: %w", pk, err) + } + + slog.Debug("Compacting partition entries", + "tp_table", pk.tpTable, + "tp_partition", pk.tpPartition, + "tp_index", pk.tpIndex, + "year", pk.year, + "month", pk.month, + "unorderedRanges", len(rm.unorderedRanges), + ) + + // func to update status with number of rows compacted for this partition key + // - passed to orderPartitionKey + updateRowsFunc := func(rowsCompacted int64) { + status.RowsCompacted += rowsCompacted + if status.TotalRows > 0 { + status.UpdateProgress() + } + updateFunc(*status) + } + + if err := orderPartitionKey(ctx, tx, pk, rm, updateRowsFunc, reindex, columns); err != nil { + slog.Error("failed to compact partition", "partition", pk, "error", err) + txErr := tx.Rollback() + if txErr != nil { + slog.Error("failed to rollback transaction after compaction", "partition", pk, "error", txErr) + } + return nil, err + } + + if err := tx.Commit(); err != nil { + slog.Error("failed to commit transaction after compaction", "partition", pk, "error", err) + txErr := tx.Rollback() + if txErr != nil { + slog.Error("failed to rollback transaction after compaction", "partition", pk, "error", txErr) + } + return nil, err + } + + slog.Info("Compacted and ordered all partition entries", + "tp_table", pk.tpTable, + "tp_partition", pk.tpPartition, + "tp_index", pk.tpIndex, + "year", pk.year, + "month", pk.month, + "input_files", pk.fileCount, + ) + + } + + slog.Info("Finished ordering DuckLake data file") + return status, nil +} + +// getColumns retrieves column information for a table, checking the map first and reading from metadata if not present +func getColumns(ctx context.Context, db *DuckDb, table string, columns map[string][]string) ([]string, error) { + // Check if columns are already cached + if cachedColumns, exists := columns[table]; exists { + return cachedColumns, nil + } + + // Read top level columns from DuckLake metadata + query := fmt.Sprintf(` + select c.column_name + from %s.ducklake_column c + join %s.ducklake_table t on c.table_id = t.table_id + where t.table_name = ? + and t.end_snapshot is null + and c.end_snapshot is null + and c.parent_column is null + order by c.column_order`, constants.DuckLakeMetadataCatalog, constants.DuckLakeMetadataCatalog) + + rows, err := db.QueryContext(ctx, query, table) + if err != nil { + return nil, fmt.Errorf("failed to get columns for table %s: %w", table, err) + } + defer rows.Close() + + var columnNames []string + for rows.Next() { + var columnName string + if err := rows.Scan(&columnName); err != nil { + return nil, fmt.Errorf("failed to scan column: %w", err) + } + columnNames = append(columnNames, columnName) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error reading columns: %w", err) + } + + // Cache the columns for future use + columns[table] = columnNames + + // and return + return columnNames, nil +} + +// orderPartitionKey processes overlapping time ranges for a partition key: +// - iterates over each unordered time range +// - reorders all data within each time range (potentially in chunks for large ranges) +// - deletes original unordered entries for that time range +func orderPartitionKey(ctx context.Context, tx *sql.Tx, pk *partitionKey, rm *reorderMetadata, updateRowsCompactedFunc func(int64), reindex bool, columns []string) error { + + slog.Debug("partition statistics", + "tp_table", pk.tpTable, + "tp_partition", pk.tpPartition, + "tp_index", pk.tpIndex, + "year", pk.year, + "month", pk.month, + "row_count", rm.rowCount, + "total file_count", pk.fileCount, + "min_timestamp", rm.minTimestamp, + "max_timestamp", rm.maxTimestamp, + "total_ranges", len(rm.unorderedRanges), + ) + + // Process each overlapping time range + for i, timeRange := range rm.unorderedRanges { + slog.Debug("processing overlapping time range", + "range_index", i+1, + "start_time", timeRange.StartTime, + "end_time", timeRange.EndTime, + "row_count", timeRange.RowCount) + + // Use the pre-calculated time range and row count from the struct + minTime := timeRange.StartTime + maxTime := timeRange.EndTime + rowCount := timeRange.RowCount + + // Determine chunking strategy for this time range + chunks, intervalDuration := determineChunkingInterval(minTime, maxTime, rowCount) + + slog.Debug("processing time range in chunks", + "range_index", i+1, + "row_count", rowCount, + "chunks", chunks, + "interval_duration", intervalDuration.String()) + + // Process this time range in chunks + currentStart := minTime + for i := 1; currentStart.Before(maxTime); i++ { + currentEnd := currentStart.Add(intervalDuration) + if currentEnd.After(maxTime) { + currentEnd = maxTime + } + + // For the final chunk, make it inclusive to catch the last row + isFinalChunk := currentEnd.Equal(maxTime) + + rowsInserted, err := insertOrderedDataForTimeRange(ctx, tx, pk, currentStart, currentEnd, isFinalChunk, reindex, columns) + if err != nil { + return fmt.Errorf("failed to insert ordered data for time range %s to %s: %w", + currentStart.Format("2006-01-02 15:04:05"), + currentEnd.Format("2006-01-02 15:04:05"), err) + } + updateRowsCompactedFunc(rowsInserted) + slog.Debug(fmt.Sprintf("processed chunk %d/%d for range %d", i, chunks, i+1)) + + // Ensure next chunk starts exactly where this one ended to prevent gaps + currentStart = currentEnd + } + + // Delete original unordered entries for this time range + err := deleteUnorderedEntriesForTimeRange(ctx, tx, rm, minTime, maxTime) + if err != nil { + return fmt.Errorf("failed to delete unordered entries for time range: %w", err) + } + + slog.Debug("completed time range", + "range_index", i+1) + } + + return nil +} + +// insertOrderedDataForTimeRange inserts ordered data for a specific time range within a partition key +func insertOrderedDataForTimeRange(ctx context.Context, tx *sql.Tx, pk *partitionKey, startTime, endTime time.Time, isFinalChunk, reindex bool, columns []string) (int64, error) { + // sanitize table name + tableName, err := backend.SanitizeDuckDBIdentifier(pk.tpTable) + if err != nil { + return 0, err + } + + // Build column list for insert + insertColumns := strings.Join(columns, ", ") + + // Build select fields + selectFields := insertColumns + // For reindexing, replace tp_index with the partition config column + if reindex && pk.partitionConfig != nil { + selectFields = strings.ReplaceAll(selectFields, "tp_index", fmt.Sprintf("%s as tp_index", pk.partitionConfig.TpIndexColumn)) + } + // For the final chunk, use inclusive end time to catch the last row + timeEndOperator := "<" + if isFinalChunk { + timeEndOperator = "<=" + } + + //nolint: gosec // sanitized + insertQuery := fmt.Sprintf(`insert into %s (%s) + select %s + from %s + where tp_timestamp >= ? + and tp_timestamp %s ? + and tp_partition = ? + and tp_index = ? + order by tp_timestamp`, + tableName, + insertColumns, + selectFields, + tableName, + timeEndOperator) + // For overlapping files, we need to reorder ALL rows in the overlapping time range + args := []interface{}{startTime, endTime, pk.tpPartition, pk.tpIndex} + + result, err := tx.ExecContext(ctx, insertQuery, args...) + if err != nil { + return 0, fmt.Errorf("failed to insert ordered data for time range: %w", err) + } + rowsInserted, err := result.RowsAffected() + if err != nil { + return 0, fmt.Errorf("failed to get rows affected count: %w", err) + } + return rowsInserted, nil +} + +// deleteUnorderedEntriesForTimeRange deletes the original unordered entries for a specific time range within a partition key +func deleteUnorderedEntriesForTimeRange(ctx context.Context, tx *sql.Tx, rm *reorderMetadata, startTime, endTime time.Time) error { + // Delete all rows in the time range for this partition key (we're re-inserting them in order) + tableName, err := backend.SanitizeDuckDBIdentifier(rm.pk.tpTable) + if err != nil { + return err + } + //nolint: gosec // sanitized + deleteQuery := fmt.Sprintf(`delete from %s + where tp_partition = ? + and tp_index = ? + and tp_timestamp >= ? + and tp_timestamp <= ? + and rowid <= ?`, + tableName) + + args := []interface{}{rm.pk.tpPartition, rm.pk.tpIndex, startTime, endTime, rm.maxRowId} + + _, err = tx.ExecContext(ctx, deleteQuery, args...) + if err != nil { + return fmt.Errorf("failed to delete unordered entries for time range: %w", err) + } + + return nil +} + +// determineChunkingInterval calculates the optimal chunking strategy for a time range based on row count. +// It returns the number of chunks and the duration of each chunk interval. +// For large datasets, it splits the time range into multiple chunks to stay within maxCompactionRowsPerChunk. +// Ensures minimum chunk interval is at least 1 hour to avoid excessive fragmentation. +func determineChunkingInterval(startTime, endTime time.Time, rowCount int64) (chunks int, intervalDuration time.Duration) { + intervalDuration = endTime.Sub(startTime) + chunks = 1 + + // If row count is greater than maxCompactionRowsPerChunk, calculate appropriate chunk interval + if rowCount > maxCompactionRowsPerChunk { + chunks = int((rowCount + maxCompactionRowsPerChunk - 1) / maxCompactionRowsPerChunk) + intervalDuration = intervalDuration / time.Duration(chunks) + + // Ensure minimum interval is at least 1 hour + if intervalDuration < time.Hour { + intervalDuration = time.Hour + } + } + + return chunks, intervalDuration +} diff --git a/internal/database/compaction_status.go b/internal/database/compaction_status.go new file mode 100644 index 00000000..5b345f31 --- /dev/null +++ b/internal/database/compaction_status.go @@ -0,0 +1,195 @@ +package database + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/dustin/go-humanize" + "github.com/turbot/go-kit/types" + "github.com/turbot/pipe-fittings/v2/backend" + "github.com/turbot/pipe-fittings/v2/constants" + "github.com/turbot/pipe-fittings/v2/utils" +) + +type CompactionStatus struct { + Message string + InitialFiles int + FinalFiles int + RowsCompacted int64 + RowsToCompact int64 + TotalRows int64 + ProgressPercent float64 + + MigrateSource int // number of source files migrated + MigrateDest int // number of destination files after migration + PartitionIndexExpressions map[string]string // the index expression used for migration for each partition + Duration time.Duration // duration of the compaction process +} + +func NewCompactionStatus() *CompactionStatus { + return &CompactionStatus{ + PartitionIndexExpressions: make(map[string]string), + } +} + +func (s *CompactionStatus) VerboseString() string { + var migratedString string + // Show migration status for each partition if any + if s.MigrateSource > 0 { + migratedString = fmt.Sprintf(`Migrated tp_index for %d %s`, + len(s.PartitionIndexExpressions), + utils.Pluralize("partition", len(s.PartitionIndexExpressions)), + ) + if s.MigrateSource != s.MigrateDest { + migratedString += fmt.Sprintf(" (%d %s migrated to %d %s)", + s.MigrateSource, + utils.Pluralize("file", s.MigrateSource), + s.MigrateDest, + utils.Pluralize("file", s.MigrateDest)) + } + migratedString += ".\n" + } + + var compactedString string + if s.RowsCompacted == 0 { + compactedString = "\nNo files required compaction." + } else { + // if the file count is the same, we must have just ordered + if s.InitialFiles == s.FinalFiles { + compactedString = fmt.Sprintf("Ordered %s rows in %s files (%s).\n", s.TotalRowsString(), s.InitialFilesString(), s.DurationString()) + } else { + compactedString = fmt.Sprintf("Compacted and ordered %s rows in %s files into %s files in (%s).\n", s.TotalRowsString(), s.InitialFilesString(), s.FinalFilesString(), s.DurationString()) + } + } + + return migratedString + compactedString +} + +func (s *CompactionStatus) String() string { + var migratedString string + var compactedString string + if s.RowsCompacted == 0 { + compactedString = "No files required compaction." + } else { + // if the file count is the same, we must have just ordered + if s.InitialFiles == s.FinalFiles { + compactedString = fmt.Sprintf("Ordered %s rows in %s files in %s.\n", s.TotalRowsString(), s.InitialFilesString(), s.Duration.String()) + } else { + compactedString = fmt.Sprintf("Compacted and ordered %s rows in %s files into %s files in %s.\n", s.TotalRowsString(), s.InitialFilesString(), s.FinalFilesString(), s.Duration.String()) + } + } + + return migratedString + compactedString +} + +func (s *CompactionStatus) TotalRowsString() any { + return humanize.Comma(s.TotalRows) +} +func (s *CompactionStatus) InitialFilesString() any { + return humanize.Comma(int64(s.InitialFiles)) +} +func (s *CompactionStatus) FinalFilesString() any { + return humanize.Comma(int64(s.FinalFiles)) +} +func (s *CompactionStatus) DurationString() string { + return utils.HumanizeDuration(s.Duration) +} +func (s *CompactionStatus) RowsCompactedString() any { + return humanize.Comma(s.RowsCompacted) +} +func (s *CompactionStatus) ProgressPercentString() string { + return fmt.Sprintf("%.1f%%", s.ProgressPercent) +} + +func (s *CompactionStatus) UpdateProgress() { + // calc percentage from RowsToCompact but print TotalRows in status message + s.ProgressPercent = (float64(s.RowsCompacted) / float64(s.RowsToCompact)) * 100 + s.Message = fmt.Sprintf(" (%0.1f%% of %s rows)", s.ProgressPercent, types.ToHumanisedString(s.TotalRows)) + +} + +func (s *CompactionStatus) getInitialCounts(ctx context.Context, db *DuckDb, partitionKeys []*partitionKey) error { + partitionNameMap := make(map[string]map[string]struct{}) + for _, pk := range partitionKeys { + s.InitialFiles += pk.fileCount + if partitionNameMap[pk.tpTable] == nil { + partitionNameMap[pk.tpTable] = make(map[string]struct{}) + } + partitionNameMap[pk.tpTable][pk.tpPartition] = struct{}{} + } + + // get row count for each table + totalRows := int64(0) + for tpTable, tpPartitions := range partitionNameMap { + + // Sanitize partition values for SQL injection protection + sanitizedPartitions := make([]string, 0, len(tpPartitions)) + for partition := range tpPartitions { + sp, err := backend.SanitizeDuckDBIdentifier(partition) + if err != nil { + return fmt.Errorf("failed to sanitize partition %s: %w", partition, err) + } + // Quote the sanitized partition name for the IN clause + sanitizedPartitions = append(sanitizedPartitions, fmt.Sprintf("'%s'", sp)) + } + + tableName, err := backend.SanitizeDuckDBIdentifier(tpTable) + if err != nil { + return fmt.Errorf("failed to sanitize table name %s: %w", tpTable, err) + } + + query := fmt.Sprintf("select count(*) from %s where tp_partition in (%s)", + tableName, + strings.Join(sanitizedPartitions, ", ")) + + var tableRowCount int64 + err = db.QueryRowContext(ctx, query).Scan(&tableRowCount) + if err != nil { + return fmt.Errorf("failed to get row count for table %s: %w", tpTable, err) + } + + totalRows += tableRowCount + } + + s.TotalRows = totalRows + return nil +} + +func (s *CompactionStatus) getFinalFileCounts(ctx context.Context, db *DuckDb, partitionKeys []*partitionKey) error { + // Get unique table names from partition keys + tableNames := make(map[string]struct{}) + for _, pk := range partitionKeys { + tableNames[pk.tpTable] = struct{}{} + } + + // Count files for each table from metadata + totalFileCount := 0 + for tableName := range tableNames { + // Sanitize table name + sanitizedTableName, err := backend.SanitizeDuckDBIdentifier(tableName) + if err != nil { + return fmt.Errorf("failed to sanitize table name %s: %w", tableName, err) + } + + // Query to count files for this table from DuckLake metadata + query := fmt.Sprintf(`select count(*) from %s.ducklake_data_file df + join %s.ducklake_table t on df.table_id = t.table_id + where t.table_name = '%s' and df.end_snapshot is null`, + constants.DuckLakeMetadataCatalog, + constants.DuckLakeMetadataCatalog, + sanitizedTableName) + + var tableFileCount int + err = db.QueryRowContext(ctx, query).Scan(&tableFileCount) + if err != nil { + return fmt.Errorf("failed to get file count for table %s: %w", tableName, err) + } + + totalFileCount += tableFileCount + } + + s.FinalFiles = totalFileCount + return nil +} diff --git a/internal/database/compaction_types.go b/internal/database/compaction_types.go new file mode 100644 index 00000000..2fc88d87 --- /dev/null +++ b/internal/database/compaction_types.go @@ -0,0 +1,178 @@ +package database + +import ( + "context" + "fmt" + "strings" + "time" +) + +// getTimeRangesToReorder analyzes file fragmentation and creates disorder metrics for a partition key. +// It queries DuckLake metadata to get all files for the partition, their timestamp ranges, and row counts. +// Then it identifies groups of files with overlapping time ranges that need compaction. +// Returns metrics including total file count and overlapping file sets with their metadata. +func getTimeRangesToReorder(ctx context.Context, db *DuckDb, pk *partitionKey, reindex bool) (*reorderMetadata, error) { + // NOTE: if we are reindexing, we must rewrite the entire partition key + // - return a single range for the entire partition key + if reindex { + rm, err := newReorderMetadata(ctx, db, pk) + if err != nil { + return nil, fmt.Errorf("failed to retrieve stats for partition key: %w", err) + } + + // make a single time range + rm.unorderedRanges = []unorderedDataTimeRange{ + { + StartTime: rm.minTimestamp, + EndTime: rm.maxTimestamp, + RowCount: rm.rowCount, + }, + } + + return rm, nil + } + + // first query the metadata to get a list of files, their timestamp ranges and row counts for this partition key + fileRanges, err := getFileRangesForPartitionKey(ctx, db, pk) + if err != nil { + return nil, fmt.Errorf("failed to get file ranges for partition key: %w", err) + } + + // Now identify which of these ranges overlap and for each overlapping set, build a superset time range + unorderedRanges, err := pk.findOverlappingFileRanges(fileRanges) + if err != nil { + return nil, fmt.Errorf("failed to build unordered time ranges: %w", err) + } + + // if there are no unordered ranges, return nil + if len(unorderedRanges) == 0 { + return nil, nil + } + + // get stats for the partition key + rm, err := newReorderMetadata(ctx, db, pk) + if err != nil { + return nil, fmt.Errorf("failed to retrieve stats for partition key: %w", err) + } + rm.unorderedRanges = unorderedRanges + return rm, nil + +} + +// query the metadata to get a list of files, their timestamp ranges and row counts for this partition key +func getFileRangesForPartitionKey(ctx context.Context, db *DuckDb, pk *partitionKey) ([]fileTimeRange, error) { + query := `select + df.path, + cast(fcs.min_value as timestamp) as min_timestamp, + cast(fcs.max_value as timestamp) as max_timestamp, + df.record_count + from __ducklake_metadata_tailpipe_ducklake.ducklake_data_file df + join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv1 + on df.data_file_id = fpv1.data_file_id and fpv1.partition_key_index = 0 + join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv2 + on df.data_file_id = fpv2.data_file_id and fpv2.partition_key_index = 1 + join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv3 + on df.data_file_id = fpv3.data_file_id and fpv3.partition_key_index = 2 + join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv4 + on df.data_file_id = fpv4.data_file_id and fpv4.partition_key_index = 3 + join __ducklake_metadata_tailpipe_ducklake.ducklake_table t + on df.table_id = t.table_id + join __ducklake_metadata_tailpipe_ducklake.ducklake_file_column_stats fcs + on df.data_file_id = fcs.data_file_id + and df.table_id = fcs.table_id + join __ducklake_metadata_tailpipe_ducklake.ducklake_column c + on fcs.column_id = c.column_id + and fcs.table_id = c.table_id + where t.table_name = ? + and fpv1.partition_value = ? + and fpv2.partition_value = ? + and fpv3.partition_value = ? + and fpv4.partition_value = ? + and c.column_name = 'tp_timestamp' + and df.end_snapshot is null + and c.end_snapshot is null + order by df.data_file_id` + + rows, err := db.QueryContext(ctx, query, pk.tpTable, pk.tpPartition, pk.tpIndex, pk.year, pk.month) + if err != nil { + return nil, fmt.Errorf("failed to get file timestamp ranges: %w", err) + } + defer rows.Close() + + var fileRanges []fileTimeRange + for rows.Next() { + var path string + var minTime, maxTime time.Time + var rowCount int64 + if err := rows.Scan(&path, &minTime, &maxTime, &rowCount); err != nil { + return nil, fmt.Errorf("failed to scan file range: %w", err) + } + fileRanges = append(fileRanges, fileTimeRange{path: path, min: minTime, max: maxTime, rowCount: rowCount}) + } + + totalFiles := len(fileRanges) + if totalFiles <= 1 { + return nil, nil + } + + // build string for the ranges + var rangesStr strings.Builder + for i, file := range fileRanges { + rangesStr.WriteString(fmt.Sprintf("start: %s, end: %s", file.min.String(), file.max.String())) + if i < len(fileRanges)-1 { + rangesStr.WriteString(", ") + } + } + return fileRanges, nil +} + +type fileTimeRange struct { + path string + min time.Time + max time.Time + rowCount int64 +} + +// unorderedDataTimeRange represents a time range containing unordered data that needs reordering +type unorderedDataTimeRange struct { + StartTime time.Time // start of the time range containing unordered data + EndTime time.Time // end of the time range containing unordered data + RowCount int64 // total row count in this time range +} + +// newUnorderedDataTimeRange creates a single unorderedDataTimeRange from overlapping files +func newUnorderedDataTimeRange(overlappingFiles []fileTimeRange) (unorderedDataTimeRange, error) { + var rowCount int64 + var startTime, endTime time.Time + + // Single loop to sum row counts and calculate time range + for i, file := range overlappingFiles { + rowCount += file.rowCount + + // Calculate time range + if i == 0 { + startTime = file.min + endTime = file.max + } else { + if file.min.Before(startTime) { + startTime = file.min + } + if file.max.After(endTime) { + endTime = file.max + } + } + } + + return unorderedDataTimeRange{ + StartTime: startTime, + EndTime: endTime, + RowCount: rowCount, + }, nil +} + +// rangesOverlap checks if two timestamp ranges overlap (excluding contiguous ranges) +func rangesOverlap(r1, r2 fileTimeRange) bool { + // Two ranges overlap if one starts before the other ends AND they're not just touching + // Contiguous ranges (where one ends exactly when the other starts) are NOT considered overlapping + return r1.min.Before(r2.max) && r2.min.Before(r1.max) +} diff --git a/internal/database/conversion_error.go b/internal/database/conversion_error.go new file mode 100644 index 00000000..d4c8608a --- /dev/null +++ b/internal/database/conversion_error.go @@ -0,0 +1,103 @@ +package database + +import ( + "bytes" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "strings" +) + +// handleConversionError attempts to handle conversion errors by counting the number of lines in the files. +// if we fail, just return the raw error. +func handleConversionError(message string, err error, paths ...string) error { + logArgs := []any{ + "error", + err, + "path", + paths, + } + + // try to count the number of rows in the file + rows, countErr := countLinesForFiles(paths...) + if countErr == nil { + logArgs = append(logArgs, "rows_affected", rows) + } + + // log error (if this is NOT a memory error + // memory errors are handles separately and retried + if !conversionRanOutOfMemory(err) { + slog.Error("parquet conversion failed", logArgs...) + } + + // return wrapped error + return NewConversionError(fmt.Errorf("%s: %w", message, err), rows, paths...) +} +func countLinesForFiles(filenames ...string) (int64, error) { + total := 0 + for _, filename := range filenames { + count, err := countLines(filename) + if err != nil { + return 0, fmt.Errorf("failed to count lines in %s: %w", filename, err) + } + total += int(count) + } + return int64(total), nil +} +func countLines(filename string) (int64, error) { + file, err := os.Open(filename) + if err != nil { + return 0, err + } + defer file.Close() + + buf := make([]byte, 64*1024) + count := 0 + + for { + c, err := file.Read(buf) + if c > 0 { + count += bytes.Count(buf[:c], []byte{'\n'}) + } + if err != nil { + if err == io.EOF { + return int64(count), nil + } + return 0, err + } + } +} + +type ConversionError struct { + SourceFiles []string + BaseError error + RowsAffected int64 + displayError string +} + +func NewConversionError(err error, rowsAffected int64, paths ...string) *ConversionError { + sourceFiles := make([]string, len(paths)) + for i, path := range paths { + sourceFiles[i] = filepath.Base(path) + } + return &ConversionError{ + SourceFiles: sourceFiles, + BaseError: err, + RowsAffected: rowsAffected, + displayError: strings.Split(err.Error(), "\n")[0], + } +} + +func (c *ConversionError) Error() string { + return fmt.Sprintf("%s: %s", strings.Join(c.SourceFiles, ", "), c.displayError) +} + +// Merge adds a second error to the conversion error message. +func (c *ConversionError) Merge(err error) { + c.BaseError = fmt.Errorf("%w: %w", c.BaseError, err) +} +func (c *ConversionError) Unwrap() error { + return c.BaseError +} diff --git a/internal/database/conversion_error_test.go b/internal/database/conversion_error_test.go new file mode 100644 index 00000000..04e022b2 --- /dev/null +++ b/internal/database/conversion_error_test.go @@ -0,0 +1,30 @@ +package database + +import ( + "errors" + "testing" +) + +func TestConversionError_WrapsRowValidationError(t *testing.T) { + // Arrange + rve := NewRowValidationError(5, []string{"colA", "colB"}) + convErr := NewConversionError(rve, 5, "/path/to/file.jsonl") + + // Act: check with errors.Is + match := errors.Is(convErr, &RowValidationError{}) + + // Assert + if !match { + t.Fatalf("expected errors.Is to match *RowValidationError, but got false") + } + + // Also: errors.As to extract + var out *RowValidationError + if !errors.As(convErr, &out) { + t.Fatalf("expected errors.As to extract *RowValidationError, but failed") + } + + if out.failedRows != 5 { + t.Errorf("unexpected failedRows: got %d, want 5", out.failedRows) + } +} diff --git a/internal/database/convertor.go b/internal/database/convertor.go new file mode 100644 index 00000000..36e873f5 --- /dev/null +++ b/internal/database/convertor.go @@ -0,0 +1,296 @@ +package database + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "sync" + "sync/atomic" + + "github.com/turbot/pipe-fittings/v2/backend" + "github.com/turbot/tailpipe-plugin-sdk/schema" + "github.com/turbot/tailpipe/internal/config" +) + +const chunkBufferLength = 1000 + +// Converter struct executes all the conversions for a single collection +// it therefore has a unique execution executionId, and will potentially convert of multiple JSONL files +// each file is assumed to have the filename format _.jsonl +// so when new input files are available, we simply store the chunk number +type Converter struct { + // the execution executionId + executionId string + + // the file scheduledChunks numbers available to process + scheduledChunks []int32 + + scheduleLock sync.Mutex + processLock sync.Mutex + + // waitGroup to track job completion + // this is incremented when a file is scheduled and decremented when the file is processed + wg sync.WaitGroup + + // the number of jsonl files processed so far + //fileCount int32 + + // the number of conversions executed + //conversionCount int32 + + // the number of rows written + rowCount int64 + // the number of rows which were NOT converted due to conversion errors encountered + failedRowCount int64 + + // the source file location + sourceDir string + // the dest file location + destDir string + + // the format string for the query to read the JSON scheduledChunks - this is reused for all scheduledChunks, + // with just the filename being added when the query is executed + readJsonQueryFormat string + + // the table conversionSchema - populated when the first chunk arrives if the conversionSchema is not already complete + conversionSchema *schema.ConversionSchema + // the source schema - which may be partial - used to build the full conversionSchema + // we store separately for the purpose of change detection + tableSchema *schema.TableSchema + + // viewQueryOnce ensures the schema inference only happens once for the first chunk, + // even if multiple scheduledChunks arrive concurrently. Combined with schemaWg, this ensures + // all subsequent scheduledChunks wait for the initial schema inference to complete before proceeding. + viewQueryOnce sync.Once + // schemaWg is used to block processing of subsequent scheduledChunks until the initial + // schema inference is complete. This ensures all scheduledChunks wait for the schema + // to be fully initialized before proceeding with their processing. + schemaWg sync.WaitGroup + + // the partition being collected + Partition *config.Partition + // func which we call with updated row count + statusFunc func(int64, int64, ...error) + + // the DuckDB database connection - this must have a ducklake attachment + db *DuckDb +} + +func NewParquetConverter(ctx context.Context, cancel context.CancelFunc, executionId string, partition *config.Partition, sourceDir string, tableSchema *schema.TableSchema, statusFunc func(int64, int64, ...error), db *DuckDb) (*Converter, error) { + // get the data dir - this will already have been created by the config loader + destDir := config.GlobalWorkspaceProfile.GetDataDir() + + // normalise the table schema to use lowercase column names + tableSchema.NormaliseColumnTypes() + + w := &Converter{ + executionId: executionId, + scheduledChunks: make([]int32, 0, chunkBufferLength), // Pre-allocate reasonable capacity + Partition: partition, + sourceDir: sourceDir, + destDir: destDir, + tableSchema: tableSchema, + statusFunc: statusFunc, + db: db, + } + + // done + return w, nil +} + +// AddChunk adds a new chunk to the list of scheduledChunks to be processed +// if this is the first chunk, determine if we have a full conversionSchema yet and if not infer from the chunk +// signal the scheduler that `scheduledChunks are available +func (w *Converter) AddChunk(executionId string, chunk int32) error { + var err error + + // wait on the schemaWg to ensure that schema inference is complete before processing the chunk + w.schemaWg.Wait() + + // Execute schema inference exactly once for the first chunk. + // The WaitGroup ensures all subsequent scheduledChunks wait for this to complete. + // If schema inference fails, the error is captured and returned to the caller. + w.viewQueryOnce.Do(func() { + err = w.onFirstChunk(executionId, chunk) + }) + if err != nil { + return fmt.Errorf("failed to infer schema: %w", err) + } + + // lock the schedule lock to ensure that we can safely add to the scheduled scheduledChunks + w.scheduleLock.Lock() + // add to scheduled scheduledChunks + w.scheduledChunks = append(w.scheduledChunks, chunk) + w.scheduleLock.Unlock() + + // increment the wait group to track the scheduled chunk + w.wg.Add(1) + + // ok try to lock the process lock - that will fail if another process is running + if w.processLock.TryLock() { + // and process = we now have the process lock + // NOTE: process chunks will keep processing as long as there are scheduledChunks to process, including + // scheduledChunks that were scheduled while we were processing + go w.processAllChunks() + } + + return nil +} + +// getChunksToProcess returns the chunks to process, up to a maximum of maxChunksToProcess +// it also trims the scheduledChunks to remove the processed chunks +func (w *Converter) getChunksToProcess() []int32 { + // now determine if there are more chunks to process + w.scheduleLock.Lock() + defer w.scheduleLock.Unlock() + + // provide a mechanism to limit the max chunks we process at once + // a high value for this seems fine (it's possible we do not actually need a limit at all) + const maxChunksToProcess = 2000 + var chunksToProcess []int32 + if len(w.scheduledChunks) > maxChunksToProcess { + slog.Debug("Converter.AddChunk limiting chunks to process to max", "scheduledChunks", len(w.scheduledChunks), "maxChunksToProcess", maxChunksToProcess) + chunksToProcess = w.scheduledChunks[:maxChunksToProcess] + // trim the scheduled chunks to remove the processed chunks + w.scheduledChunks = w.scheduledChunks[maxChunksToProcess:] + } else { + slog.Debug("Converter.AddChunk processing all scheduled chunks", "scheduledChunks", len(w.scheduledChunks)) + chunksToProcess = w.scheduledChunks + // clear the scheduled chunks + w.scheduledChunks = nil + } + return chunksToProcess +} + +// onFirstChunk is called when the first chunk is added to the converter +// it is responsible for building the conversion schema if it does not already exist +// (we must wait for the first chunk as we may need to infer the schema from the chunk data) +// once the conversion schema is built, we can create the DuckDB table for this partition and build the +// read query format string that we will use to read the JSON data from the file +func (w *Converter) onFirstChunk(executionId string, chunk int32) error { + w.schemaWg.Add(1) + defer w.schemaWg.Done() + if err := w.buildConversionSchema(executionId, chunk); err != nil { + // err will be returned by the parent function + return err + } + // create the DuckDB table for this partition if it does not already exist + if err := EnsureDuckLakeTable(w.conversionSchema.Columns, w.db, w.Partition.TableName); err != nil { + return fmt.Errorf("failed to create DuckDB table: %w", err) + } + w.readJsonQueryFormat = buildReadJsonQueryFormat(w.conversionSchema, w.Partition) + + return nil +} + +// WaitForConversions waits for all jobs to be processed or for the context to be cancelled +func (w *Converter) WaitForConversions(ctx context.Context) error { + slog.Info("Converter.WaitForConversions - waiting for all jobs to be processed or context to be cancelled.") + // wait for the wait group within a goroutine so we can also check the context + done := make(chan struct{}) + go func() { + w.wg.Wait() + close(done) + }() + + select { + case <-ctx.Done(): + slog.Info("WaitForConversions - context cancelled.") + return ctx.Err() + case <-done: + slog.Info("WaitForConversions - all jobs processed.") + return nil + } +} + +// addJobErrors calls the status func with any job errors, first summing the failed rows in any conversion errors +func (w *Converter) addJobErrors(errorList ...error) { + var failedRowCount int64 + + for _, err := range errorList { + var conversionError = &ConversionError{} + if errors.As(err, &conversionError) { + failedRowCount = atomic.AddInt64(&w.failedRowCount, conversionError.RowsAffected) + } + slog.Error("conversion error", "error", err) + } + + // update the status function with the new error count (no need to use atomic for errorList as we are already locked) + w.statusFunc(atomic.LoadInt64(&w.rowCount), failedRowCount, errorList...) +} + +// updateRowCount atomically increments the row count and calls the statusFunc +func (w *Converter) updateRowCount(count int64) { + atomic.AddInt64(&w.rowCount, count) + // call the status function with the new row count + w.statusFunc(atomic.LoadInt64(&w.rowCount), atomic.LoadInt64(&w.failedRowCount)) +} + +// CheckTableSchema checks if the specified table exists in the DuckDB database and compares its schema with the +// provided schema. +// it returns a TableSchemaStatus indicating whether the table exists, whether the schema matches, and any differences. +// THis is not used at present but will be used when we implement ducklake schema evolution handling +func (w *Converter) CheckTableSchema(db *sql.DB, tableName string, conversionSchema schema.ConversionSchema) (TableSchemaStatus, error) { + // Check if table exists + exists, err := w.tableExists(db, tableName) + if err != nil { + return TableSchemaStatus{}, err + } + + if !exists { + return TableSchemaStatus{}, nil + } + + // Get existing schema + existingSchema, err := w.getTableSchema(db, tableName) + if err != nil { + return TableSchemaStatus{}, fmt.Errorf("failed to retrieve schema: %w", err) + } + + // Use constructor to create status from comparison + diff := NewTableSchemaStatusFromComparison(existingSchema, conversionSchema) + return diff, nil +} + +func (w *Converter) tableExists(db *sql.DB, tableName string) (bool, error) { + sanitizedTableName, err := backend.SanitizeDuckDBIdentifier(tableName) + if err != nil { + return false, fmt.Errorf("invalid table name %s: %w", tableName, err) + } + //nolint:gosec // table name is sanitized + query := fmt.Sprintf("select exists (select 1 from information_schema.tables where table_name = '%s')", sanitizedTableName) + var exists int + if err := db.QueryRow(query).Scan(&exists); err != nil { + return false, err + } + return exists == 1, nil +} + +func (w *Converter) getTableSchema(db *sql.DB, tableName string) (map[string]schema.ColumnSchema, error) { + query := fmt.Sprintf("pragma table_info(%s);", tableName) + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + schemaMap := make(map[string]schema.ColumnSchema) + for rows.Next() { + var name, dataType string + var notNull, pk int + var dfltValue sql.NullString + + if err := rows.Scan(&name, &dataType, ¬Null, &dfltValue, &pk); err != nil { + return nil, err + } + + schemaMap[name] = schema.ColumnSchema{ + ColumnName: name, + Type: dataType, + } + } + + return schemaMap, nil +} diff --git a/internal/database/convertor_convert.go b/internal/database/convertor_convert.go new file mode 100644 index 00000000..3e53b389 --- /dev/null +++ b/internal/database/convertor_convert.go @@ -0,0 +1,275 @@ +package database + +import ( + "errors" + "fmt" + "log" + "log/slog" + "os" + "path/filepath" + "strings" + "time" + + "github.com/turbot/pipe-fittings/v2/utils" + + "github.com/marcboeker/go-duckdb/v2" + "github.com/turbot/tailpipe-plugin-sdk/table" +) + +// process all available chunks +// this is called when a chunk is added but will continue processing any further chunks added while we were processing +func (w *Converter) processAllChunks() { + // note we ALREADY HAVE THE PROCESS LOCK - be sure to release it when we are done + defer w.processLock.Unlock() + + // so we have the process lock AND the schedule lock + // move the scheduled chunks to the chunks to process + // (scheduledChunks may be empty, in which case we will break out of the loop) + chunksToProcess := w.getChunksToProcess() + for len(chunksToProcess) > 0 { + err := w.processChunks(chunksToProcess) + if err != nil { + slog.Error("Error processing chunks", "error", err) + // call add job errors and carry on + w.addJobErrors(err) + } + //- get next batch of chunks + chunksToProcess = w.getChunksToProcess() + } + + // if we get here, we have processed all scheduled chunks (but more may come later + log.Print("BatchProcessor: all scheduled chunks processed for execution") +} + +// process a batch of chunks +// Note whether successful of not, this decrements w.wg by the chunk count on return +func (w *Converter) processChunks(chunksToProcess []int32) error { + // decrement the wait group by the number of chunks processed + defer func() { + w.wg.Add(len(chunksToProcess) * -1) + }() + + // build a list of filenames to process + filenamesToProcess, err := w.chunkNumbersToFilenames(chunksToProcess) + if err != nil { + slog.Error("chunkNumbersToFilenames failed") + // chunkNumbersToFilenames returns a conversionError + return err + } + + // execute conversion query for the chunks + // (insertBatchIntoDuckLake will return a coinversionError) + err = w.insertBatchIntoDuckLake(filenamesToProcess) + // delete the files after processing (successful or otherwise) - we will just return err + for _, filename := range filenamesToProcess { + if deleteErr := os.Remove(filename); deleteErr != nil { + slog.Error("Failed to delete file after processing", "file", filename, "error", err) + // give conversion error precedence + if err == nil { + err = deleteErr + } + } + } + // return error (if any) + return err +} + +func (w *Converter) chunkNumbersToFilenames(chunks []int32) ([]string, error) { + var filenames = make([]string, len(chunks)) + var missingFiles []string + for i, chunkNumber := range chunks { + // build the source filename + jsonlFilePath := filepath.Join(w.sourceDir, table.ExecutionIdToJsonlFileName(w.executionId, chunkNumber)) + // verify file exists + if _, err := os.Stat(jsonlFilePath); os.IsNotExist(err) { + missingFiles = append(missingFiles, jsonlFilePath) + } + // remove single quotes from the file path to avoid issues with SQL queries + escapedPath := strings.ReplaceAll(jsonlFilePath, "'", "''") + filenames[i] = escapedPath + } + if len(missingFiles) > 0 { + // raise conversion error for the missing files - we do now know the row count so pass zero + return filenames, NewConversionError(fmt.Errorf("%s not found", + utils.Pluralize("file", len(missingFiles))), + 0, + missingFiles...) + + } + return filenames, nil +} + +func (w *Converter) insertBatchIntoDuckLake(filenames []string) (err error) { + t := time.Now() + + // copy the data from the jsonl file to a temp table + if err := w.copyChunkToTempTable(filenames); err != nil { + // copyChunkToTempTable will already have called handleSchemaChangeError anf handleConversionError + return err + } + + tempTime := time.Now() + + // now validate the data + validateRowsError := w.validateRows(filenames) + if validateRowsError != nil { + // if the error is NOT RowValidationError, just return it + // (if it is a validation error, we have special handling) + if !errors.Is(validateRowsError, &RowValidationError{}) { + return validateRowsError + } + + // so it IS a row validation error - the invalid rows will have been removed from the temp table + // - process the rest of the chunk + // ensure that we return the row validation error, merged with any other error we receive + defer func() { + if err == nil { + err = validateRowsError + } else { + // so we have an error (aside from the any validation error) + // convert the validation error to a conversion error (which will be wrapping the validation error + var conversionError *ConversionError + // we expect this will always pass + if errors.As(validateRowsError, &conversionError) { + conversionError.Merge(err) + } + err = conversionError + } + }() + } + + slog.Debug("about to insert rows into ducklake table") + + rowCount, err := w.insertIntoDucklake(w.Partition.TableName) + if err != nil { + slog.Error("failed to insert into DuckLake table", "table", w.Partition.TableName, "error", err) + return err + } + + td := tempTime.Sub(t) + cd := time.Since(tempTime) + total := time.Since(t) + + // Update counters and advance to the next batch + // if we have an error, return it below + // update the row count + w.updateRowCount(rowCount) + + slog.Debug("inserted rows into DuckLake table", "chunks", len(filenames), "row count", rowCount, "error", err, "temp time", td.Milliseconds(), "conversion time", cd.Milliseconds(), "total time ", total.Milliseconds()) + return nil +} + +func (w *Converter) copyChunkToTempTable(jsonlFilePaths []string) error { + var queryBuilder strings.Builder + + // Check for empty file paths + if len(jsonlFilePaths) == 0 { + return fmt.Errorf("no file paths provided") + } + + // Create SQL array of file paths + var fileSQL string + if len(jsonlFilePaths) == 1 { + fileSQL = fmt.Sprintf("'%s'", jsonlFilePaths[0]) + } else { + // For multiple files, create a properly quoted array + var quotedPaths []string + for _, jsonFilePath := range jsonlFilePaths { + quotedPaths = append(quotedPaths, fmt.Sprintf("'%s'", jsonFilePath)) + } + fileSQL = "[" + strings.Join(quotedPaths, ", ") + "]" + } + + // render the read JSON query with the jsonl file path + // - this build a select clause which selects the required data from the JSONL file (with columns types specified) + selectQuery := fmt.Sprintf(w.readJsonQueryFormat, fileSQL) + + // Step: Prepare the temp table from JSONL input + // + // - Drop the temp table if it exists + // - Create a new temp table by executing the dselect query + queryBuilder.WriteString(fmt.Sprintf(` +drop table if exists temp_data; + +create temp table temp_data as + %s +`, selectQuery)) + + _, err := w.db.Exec(queryBuilder.String()) + if err != nil { + // if the error is a schema change error, determine whether the schema of these chunks is + // different to the inferred schema + // w.handleSchemaChangeError either returns a schema change error or the original error + return w.handleSchemaChangeError(err, jsonlFilePaths...) + } + + return nil +} + +// insertIntoDucklakeForBatch writes a batch of rows from the temp_data table to the specified target DuckDB table. +// +// It selects rows based on rowid, using the provided startRowId and rowCount to control the range: +// - Rows with rowid > startRowId and rowid <= (startRowId + rowCount) are selected. +// +// This approach allows for efficient batching from the temporary table into the final destination table. +// +// To prevent schema mismatches, it explicitly lists columns in the INSERT statement based on the conversion schema. +// +// Returns the number of rows inserted and any error encountered. +func (w *Converter) insertIntoDucklake(targetTable string) (int64, error) { + // quote the table name + targetTable = fmt.Sprintf(`"%s"`, targetTable) + + // Build the final INSERT INTO ... SELECT statement using the fully qualified table name. + columns := w.conversionSchema.ColumnString + insertQuery := fmt.Sprintf(` + insert into %s (%s) + select %s from temp_data + `, targetTable, columns, columns) + + // Execute the insert statement + result, err := w.db.Exec(insertQuery) + if err != nil { + slog.Error(fmt.Sprintf("failed to insert data into DuckLake table db %p", w.db.DB), "table", targetTable, "error", err, "db", w.db.DB) + // It's helpful to wrap the error with context about what failed. + return 0, fmt.Errorf("failed to insert data into %s: %w", targetTable, err) + } + + // Get the number of rows that were actually inserted. + insertedRowCount, err := result.RowsAffected() + if err != nil { + return 0, fmt.Errorf("failed to get number of affected rows: %w", err) + } + + return insertedRowCount, nil +} + +// handleSchemaChangeError determines if the error is because the schema of this chunk is different to the inferred schema +// infer the schema of this chunk and compare - if they are different, return that in an error +func (w *Converter) handleSchemaChangeError(origError error, jsonlFilePaths ...string) error { + // check all files for a schema change error + for _, jsonlFilePath := range jsonlFilePaths { + err := w.detectSchemaChange(jsonlFilePath) + if err != nil { + // if the error returned from detectSchemaChange is a SchemaChangeError, return that instead of the original error + // (ignore any other error - we will fall through to return original error) + var schemaChangeError = &SchemaChangeError{} + if errors.As(err, &schemaChangeError) { + // update err and fall through to handleConversionError - this wraps the error with additional row count info + return schemaChangeError + } + } + } + + // just return the original error + return origError +} + +// conversionRanOutOfMemory checks if the error is an out-of-memory error from DuckDB +func conversionRanOutOfMemory(err error) bool { + var duckDBErr = &duckdb.Error{} + if errors.As(err, &duckDBErr) { + return duckDBErr.Type == duckdb.ErrorTypeOutOfMemory + } + return false +} diff --git a/internal/parquet/convertor_infer.go b/internal/database/convertor_schema.go similarity index 83% rename from internal/parquet/convertor_infer.go rename to internal/database/convertor_schema.go index 39e1f379..929d0795 100644 --- a/internal/parquet/convertor_infer.go +++ b/internal/database/convertor_schema.go @@ -1,21 +1,19 @@ -package parquet +package database import ( "encoding/json" "fmt" + "path/filepath" + "github.com/turbot/tailpipe-plugin-sdk/schema" "github.com/turbot/tailpipe-plugin-sdk/table" - "github.com/turbot/tailpipe/internal/database" - "log" - "path/filepath" - "strings" ) // populate the ConversionSchema // determine if we have a full schema yet and if not infer from the chunk func (w *Converter) buildConversionSchema(executionID string, chunk int32) error { - // if table schema is already complete, we can skip the inference and just populate the conversionSchema + // complete means that we have types for all columns in the table schema, and we are not mapping any source columns if w.tableSchema.Complete() { w.conversionSchema = schema.NewConversionSchema(w.tableSchema) return nil @@ -47,8 +45,8 @@ func (w *Converter) inferConversionSchema(executionId string, chunkNumber int32) } func (w *Converter) InferSchemaForJSONLFile(filePath string) (*schema.TableSchema, error) { - // TODO figure out why we need this hack - trying 2 different methods - inferredSchema, err := w.inferSchemaForJSONLFileWithDescribe(filePath) + // depending on the data we have observed that one of the two queries will work + inferredSchema, err := w.inferSchemaForJSONLFileWithDescribe(w.db, filePath) if err != nil { inferredSchema, err = w.inferSchemaForJSONLFileWithJSONStructure(filePath) } @@ -63,13 +61,6 @@ func (w *Converter) InferSchemaForJSONLFile(filePath string) (*schema.TableSchem // it uses 2 different queries as depending on the data, one or the other has been observed to work // (needs investigation) func (w *Converter) inferSchemaForJSONLFileWithJSONStructure(filePath string) (*schema.TableSchema, error) { - // Open DuckDB connection - db, err := database.NewDuckDb() - if err != nil { - log.Fatalf("failed to open DuckDB connection: %v", err) - } - defer db.Close() - // Query to infer schema using json_structure query := ` select json_structure(json)::varchar as schema @@ -78,7 +69,7 @@ func (w *Converter) inferSchemaForJSONLFileWithJSONStructure(filePath string) (* ` var schemaStr string - err = db.QueryRow(query, filePath).Scan(&schemaStr) + err := w.db.QueryRow(query, filePath).Scan(&schemaStr) if err != nil { return nil, fmt.Errorf("failed to execute query: %w", err) } @@ -106,15 +97,7 @@ func (w *Converter) inferSchemaForJSONLFileWithJSONStructure(filePath string) (* return res, nil } -func (w *Converter) inferSchemaForJSONLFileWithDescribe(filePath string) (*schema.TableSchema, error) { - - // Open DuckDB connection - db, err := database.NewDuckDb() - if err != nil { - log.Fatalf("failed to open DuckDB connection: %v", err) - } - defer db.Close() - +func (w *Converter) inferSchemaForJSONLFileWithDescribe(db *DuckDb, filePath string) (*schema.TableSchema, error) { // Use DuckDB to describe the schema of the JSONL file query := `SELECT column_name, column_type FROM (DESCRIBE (SELECT * FROM read_json_auto(?)))` @@ -149,14 +132,6 @@ func (w *Converter) inferSchemaForJSONLFileWithDescribe(filePath string) (*schem return res, nil } -func (e *SchemaChangeError) Error() string { - changeStrings := make([]string, len(e.ChangedColumns)) - for i, change := range e.ChangedColumns { - changeStrings[i] = fmt.Sprintf("'%s': '%s' -> '%s'", change.Name, change.OldType, change.NewType) - } - return fmt.Sprintf("inferred schema change detected - consider specifying a column type in table definition: %s", strings.Join(changeStrings, ", ")) -} - func (w *Converter) detectSchemaChange(filePath string) error { inferredChunksSchema, err := w.InferSchemaForJSONLFile(filePath) if err != nil { diff --git a/internal/database/convertor_validate.go b/internal/database/convertor_validate.go new file mode 100644 index 00000000..8c96d944 --- /dev/null +++ b/internal/database/convertor_validate.go @@ -0,0 +1,110 @@ +package database + +import ( + "fmt" + "strings" +) + +// validateRows validates required fields are non null +// it also validates that the schema of the chunk is the same as the inferred schema and if it is not, reports a useful error +// the query count of invalid rows and a list of null fields +func (w *Converter) validateRows(jsonlFilePaths []string) error { + // build array of required columns to validate + var requiredColumns []string + for _, col := range w.conversionSchema.Columns { + if col.Required { + // if the column is required, add it to the list of columns to validate + requiredColumns = append(requiredColumns, col.ColumnName) + } + } + + // if we have no columns to validate, biuld a validation query to return the number of invalid rows and the columns with nulls + validationQuery := w.buildValidationQuery(requiredColumns) + + row := w.db.QueryRow(validationQuery) + var failedRowCount int64 + var columnsWithNullsInterface []interface{} + + err := row.Scan(&failedRowCount, &columnsWithNullsInterface) + if err != nil { + return handleConversionError("row validation query failed", err, jsonlFilePaths...) + } + + if failedRowCount == 0 { + // no rows with nulls - we are done + return nil + } + + // delete invalid rows from the temp table + if err := w.deleteInvalidRows(requiredColumns); err != nil { + // failed to delete invalid rows - return an error + err := handleConversionError("failed to delete invalid rows from temp table", err, jsonlFilePaths...) + return err + } + + // Convert the interface slice to string slice + var columnsWithNulls []string + for _, col := range columnsWithNullsInterface { + if col != nil { + columnsWithNulls = append(columnsWithNulls, col.(string)) + } + } + + // we have a failure - return an error with details about which columns had nulls + // wrap a row validation error inside a conversion error + return NewConversionError(NewRowValidationError(failedRowCount, columnsWithNulls), failedRowCount, jsonlFilePaths...) +} + +// buildValidationQuery builds a query to copy the data from the select query to a temp table +// it then validates that the required columns are not null, removing invalid rows and returning +// the count of invalid rows and the columns with nulls +func (w *Converter) buildValidationQuery(requiredColumns []string) string { + var queryBuilder strings.Builder + + // Build the validation query that: + // - Counts distinct rows that have null values in required columns + // - Lists all required columns that contain null values + queryBuilder.WriteString(`select + count(distinct rowid) as rows_with_required_nulls, -- Count unique rows with nulls in required columns + coalesce(list(distinct col), []) as required_columns_with_nulls -- List required columns that have null values, defaulting to empty list if NULL +from (`) + + // Step 3: For each required column we need to validate: + // - Create a query that selects rows where this column is null + // - Include the column name so we know which column had the null + // - UNION ALL combines all these results (faster than UNION as we don't need to deduplicate) + for i, col := range requiredColumns { + if i > 0 { + queryBuilder.WriteString(" union all\n") + } + // For each required column, create a query that: + // - Selects the rowid (to count distinct rows) + // - Includes the column name (to list which columns had nulls) + // - Only includes rows where this column is null + queryBuilder.WriteString(fmt.Sprintf(" select rowid, '%s' as col from temp_data where %s is null\n", col, col)) + } + + queryBuilder.WriteString(");") + + return queryBuilder.String() +} + +// buildNullCheckQuery builds a WHERE clause to check for null values in the specified columns +func (w *Converter) buildNullCheckQuery(requiredColumns []string) string { + + // build a slice of null check conditions + conditions := make([]string, len(requiredColumns)) + for i, col := range requiredColumns { + conditions[i] = fmt.Sprintf("%s is null", col) + } + return strings.Join(conditions, " or ") +} + +// deleteInvalidRows removes rows with null values in the specified columns from the temp table +func (w *Converter) deleteInvalidRows(requiredColumns []string) error { + whereClause := w.buildNullCheckQuery(requiredColumns) + query := fmt.Sprintf("delete from temp_data where %s;", whereClause) + + _, err := w.db.Exec(query) + return err +} diff --git a/internal/database/create.go b/internal/database/create.go deleted file mode 100644 index 9c237152..00000000 --- a/internal/database/create.go +++ /dev/null @@ -1,27 +0,0 @@ -package database - -import ( - "context" - _ "github.com/marcboeker/go-duckdb/v2" - filehelpers "github.com/turbot/go-kit/files" - _ "github.com/turbot/go-kit/helpers" - _ "github.com/turbot/pipe-fittings/v2/utils" - "github.com/turbot/tailpipe/internal/filepaths" -) - -func EnsureDatabaseFile(ctx context.Context) error { - databaseFilePath := filepaths.TailpipeDbFilePath() - if filehelpers.FileExists(databaseFilePath) { - return nil - } - - // - // Open a DuckDB connection (creates the file if it doesn't exist) - db, err := NewDuckDb(WithDbFile(databaseFilePath)) - if err != nil { - return err - } - defer db.Close() - - return AddTableViews(ctx, db) -} diff --git a/internal/database/duck_db.go b/internal/database/duck_db.go index 9d34674b..d3033dbb 100644 --- a/internal/database/duck_db.go +++ b/internal/database/duck_db.go @@ -4,9 +4,13 @@ import ( "context" "database/sql" "fmt" + "log/slog" + "os" + "strings" + pconstants "github.com/turbot/pipe-fittings/v2/constants" pf "github.com/turbot/pipe-fittings/v2/filepaths" - "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/filepaths" ) @@ -17,44 +21,86 @@ import ( type DuckDb struct { // duckDb connection *sql.DB - extensions []string - dataSourceName string - tempDir string + extensions []string + dataSourceName string + tempDir string + maxMemoryMb int + ducklakeEnabled bool + // create a read only connection to ducklake + duckLakeReadOnly bool + + // a list of view filters - if this is set, we create a set of views in the database, one per table, + // applying the specified filter + // NOTE: if view filters are specified, the connection is set to READ ONLY mode (even if read only option is not set) + viewFilters []string } -func NewDuckDb(opts ...DuckDbOpt) (*DuckDb, error) { - w := &DuckDb{} +func NewDuckDb(opts ...DuckDbOpt) (_ *DuckDb, err error) { + slog.Info("Initializing DuckDB connection") + + d := &DuckDb{} for _, opt := range opts { - opt(w) + opt(d) } + defer func() { + if err != nil { + // If an error occurs during initialization, close the DB connection if it was opened + if d.DB != nil { + _ = d.DB.Close() + } + d.DB = nil // ensure DB is nil to avoid further operations on a closed connection + } + }() + // Connect to DuckDB - db, err := sql.Open("duckdb", w.dataSourceName) + db, err := sql.Open("duckdb", d.dataSourceName) if err != nil { return nil, fmt.Errorf("failed to open DuckDB connection: %w", err) } - w.DB = db + d.DB = db + + // for duckdb, limit connections to 1 - DuckDB is designed for single-connection usage + d.SetMaxOpenConns(1) + + // set the extension directory + if _, err := d.DB.Exec("set extension_directory = ?;", pf.EnsurePipesDuckDbExtensionsDir()); err != nil { + return nil, fmt.Errorf("failed to set extension_directory: %w", err) + } - if len(w.extensions) > 0 { - // install and load the JSON extension - if err := w.installAndLoadExtensions(); err != nil { + if len(d.extensions) > 0 { + // set extension dir and install any specified extensions + if err := d.installAndLoadExtensions(); err != nil { return nil, fmt.Errorf(": %w", err) } } + if d.ducklakeEnabled { + if err := d.connectDucklake(context.Background()); err != nil { + return nil, fmt.Errorf("failed to connect to DuckLake: %w", err) + } + } - // Configure DuckDB's temp directory: - // - If WithTempDir option was provided, use that directory - // - Otherwise, use the collection temp directory (a subdirectory in the user's home directory - // where temporary files for data collection are stored) - tempDir := w.tempDir - if tempDir == "" { - tempDir = filepaths.EnsureCollectionTempDir() + // view filters are used to create a database with a filtered set of data to query, + // used to support date filtering for the index command + if len(d.viewFilters) > 0 { + err = d.createFilteredViews(d.viewFilters) + if err != nil { + return nil, fmt.Errorf("failed to create filtered views: %w", err) + } } - if _, err := db.Exec(fmt.Sprintf("SET temp_directory = '%s';", tempDir)); err != nil { - _ = w.Close() - return nil, fmt.Errorf("failed to set temp_directory: %w", err) + + // Configure DuckDB's temp directory + if err := d.setTempDir(); err != nil { + return nil, fmt.Errorf("failed to set DuckDB temp directory: %w", err) + } + // set the max memory if specified + if d.maxMemoryMb > 0 { + if _, err := db.Exec("set max_memory = ? || 'MB';", d.maxMemoryMb); err != nil { + _ = d.Close() + return nil, fmt.Errorf("failed to set max_memory: %w", err) + } } - return w, nil + return d, nil } func (d *DuckDb) Query(query string, args ...any) (*sql.Rows, error) { @@ -95,6 +141,14 @@ func (d *DuckDb) ExecContext(ctx context.Context, query string, args ...interfac }) } +// GetTempDir returns the temporary directory configured for DuckDB operations +func (d *DuckDb) GetTempDir() string { + if d.tempDir == "" { + return filepaths.EnsureCollectionTempDir() + } + return d.tempDir +} + func (d *DuckDb) installAndLoadExtensions() error { if d.DB == nil { return fmt.Errorf("db is nil") @@ -103,13 +157,8 @@ func (d *DuckDb) installAndLoadExtensions() error { return nil } - // set the extension directory - if _, err := d.DB.Exec(fmt.Sprintf("SET extension_directory = '%s';", pf.EnsurePipesDuckDbExtensionsDir())); err != nil { - return fmt.Errorf("failed to set extension_directory: %w", err) - } - // install and load the extensions - for _, extension := range constants.DuckDbExtensions { + for _, extension := range pconstants.DuckDbExtensions { if _, err := d.DB.Exec(fmt.Sprintf("INSTALL '%s'; LOAD '%s';", extension, extension)); err != nil { return fmt.Errorf("failed to install and load extension %s: %s", extension, err.Error()) } @@ -117,3 +166,126 @@ func (d *DuckDb) installAndLoadExtensions() error { return nil } + +// connectDucklake connects the given DuckDB connection to DuckLake +func (d *DuckDb) connectDucklake(ctx context.Context) error { + // we share the same set of commands for tailpipe connection - get init commands and execute them + commands := GetDucklakeInitCommands(d.duckLakeReadOnly) + // if there are NO view filters, set the default catalog to ducklake + // if there are view filters, the views will be created in the default memory catalog so do not change the default + if len(d.viewFilters) == 0 { + commands = append(commands, SqlCommand{ + Description: "set default catalog to ducklake", + Command: fmt.Sprintf("use %s", pconstants.DuckLakeCatalog), + }) + } + + // tactical: if read only mode is set and the ducklake database does not exists, create it + // (creating a read only connection will FAIL if the ducklake database has not been created yet + // - writeable connections will create the database if it does not exist) + if d.duckLakeReadOnly { + if err := ensureDucklakeDb(); err != nil { + return fmt.Errorf("failed to ensure ducklake database exists: %w", err) + } + } + + for _, cmd := range commands { + slog.Info(cmd.Description, "command", cmd.Command) + _, err := d.ExecContext(ctx, cmd.Command) + if err != nil { + return fmt.Errorf("%s failed: %w", cmd.Description, err) + } + } + + return nil +} + +// ensureDucklakeDb checks if the ducklake database file exists, and if not, creates it by opening +// and closing a duckdb connection with ducklake enabled +// this is used if we we are creating a readonly db connection to ducklake +// - readonly connections will fail if the ducklake database does not exist +func ensureDucklakeDb() error { + //check db file exists + _, err := os.Stat(config.GlobalWorkspaceProfile.GetDucklakeDbPath()) + if err == nil { + // file exists - nothing to do + return nil + } + // create a duck db connection then close again + db, err := NewDuckDb(WithDuckLake()) + if err != nil { + return err + } + if err := db.Close(); err != nil { + return fmt.Errorf("failed to close duckdb connection: %w", err) + } + return nil + +} + +func (d *DuckDb) createFilteredViews(filters []string) error { + // get the sql to create the views based on the filters + viewSql, err := GetCreateViewsSql(context.Background(), d, d.viewFilters...) + if err != nil { + return fmt.Errorf("failed to get create views sql: %w", err) + } + // execute the commands to create the views + slog.Info("Creating views") + for _, cmd := range viewSql { + if _, err := d.Exec(cmd.Command); err != nil { + return fmt.Errorf("failed to create view: %w", err) + } + } + return nil +} + +// Configure DuckDB's temp directory +// - If WithTempDir option was provided, use that directory +// - Otherwise, use the collection temp directory (a subdirectory in the user's home directory +// where temporary files for data collection are stored) +func (d *DuckDb) setTempDir() error { + tempDir := d.tempDir + if tempDir == "" { + baseDir := filepaths.EnsureCollectionTempDir() + // Create a unique subdirectory with 'duckdb-' prefix + // it is important to use a unique directory for each DuckDB instance as otherwise temp files from + // different instances can conflict with each other, causing memory swapping issues + uniqueTempDir, err := os.MkdirTemp(baseDir, "duckdb-") + if err != nil { + return fmt.Errorf("failed to create unique temp directory: %w", err) + } + tempDir = uniqueTempDir + } + + if _, err := d.Exec("set temp_directory = ?;", tempDir); err != nil { + _ = d.Close() + return fmt.Errorf("failed to set temp_directory: %w", err) + } + return nil +} + +// GetDucklakeInitCommands returns the set of SQL commands required to initialize and connect to DuckLake. +// this is used both for tailpipe to connect to ducklake and also for tailpipe connect to build the init script +// It returns an ordered slice of SQL commands. +func GetDucklakeInitCommands(readonly bool) []SqlCommand { + attachOptions := []string{ + fmt.Sprintf("data_path '%s'", config.GlobalWorkspaceProfile.GetDataDir()), + "meta_journal_mode 'WAL'", + } + // if readonly mode is requested, add the option + if readonly { + attachOptions = append(attachOptions, "READ_ONLY") + } + attachQuery := fmt.Sprintf(`attach 'ducklake:sqlite:%s' AS %s ( + %s)`, + config.GlobalWorkspaceProfile.GetDucklakeDbPath(), + pconstants.DuckLakeCatalog, + strings.Join(attachOptions, ",\n\t")) + + commands := []SqlCommand{ + {Description: "install sqlite extension", Command: "install sqlite"}, + {Description: "install ducklake extension", Command: "install ducklake;"}, + {Description: "attach to ducklake database", Command: attachQuery}, + } + return commands +} diff --git a/internal/database/duck_db_error.go b/internal/database/duck_db_error.go index d03e0a80..693c48a5 100644 --- a/internal/database/duck_db_error.go +++ b/internal/database/duck_db_error.go @@ -7,6 +7,7 @@ import ( "os" "regexp" "sort" + "strconv" "strings" "time" @@ -81,7 +82,6 @@ func handleDuckDbError(err error) error { return newInvalidParquetError(updatedFilename) } // so we have no filename - //TODO handle Invalid Error: TProtocolException: Invalid data } return err @@ -162,21 +162,34 @@ func newInvalidParquetError(parquetFilePath string) error { parquetFilePath: parquetFilePath, } + var year, month int + // Extract table, partition and date from path components parts := strings.Split(parquetFilePath, "/") for _, part := range parts { - if strings.HasPrefix(part, "tp_table=") { + switch { + case strings.HasPrefix(part, "tp_table="): err.table = strings.TrimPrefix(part, "tp_table=") - } else if strings.HasPrefix(part, "tp_partition=") { + case strings.HasPrefix(part, "tp_partition="): err.partition = strings.TrimPrefix(part, "tp_partition=") - } else if strings.HasPrefix(part, "tp_date=") { - dateString := strings.TrimPrefix(part, "tp_date=") - date, parseErr := time.Parse("2006-01-02", dateString) + case strings.HasPrefix(part, "year="): + yearString := strings.TrimPrefix(part, "year=") + y, parseErr := strconv.Atoi(yearString) + if parseErr == nil { + year = y + } + case strings.HasPrefix(part, "month="): + monthString := strings.TrimPrefix(part, "month=") + m, parseErr := strconv.Atoi(monthString) if parseErr == nil { - err.date = date + month = m } } } + // if we have a year and month, set the error date + if year > 0 && month > 0 { + err.date = time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.UTC) + } return err } diff --git a/internal/database/duck_db_options.go b/internal/database/duck_db_options.go index a34a3df1..9b303915 100644 --- a/internal/database/duck_db_options.go +++ b/internal/database/duck_db_options.go @@ -28,4 +28,32 @@ func WithTempDir(dir string) DuckDbOpt { return func(d *DuckDb) { d.tempDir = dir } + +} + +// WithMaxMemoryMb sets the maximum memory limit for DuckDB. +// This can be used to control the memory usage of DuckDB operations. +func WithMaxMemoryMb(maxMemoryMb int) DuckDbOpt { + return func(d *DuckDb) { + d.maxMemoryMb = maxMemoryMb + } +} + +// WithDuckLake enables the DuckLake extension for DuckDB. +func WithDuckLake() DuckDbOpt { + return func(d *DuckDb) { + d.ducklakeEnabled = true + } +} + +// WithDuckLakeReadonly enables the DuckLake extension in read-only mode. +// filters is an optional list of SQL filter expressions - if specified, a view will be created for each table in the database +// and the filters will be applied to the view. +// If no filters are specified, the ducklake attachment will be set as the default catalog so the tables can be accessed directly +func WithDuckLakeReadonly(filters ...string) DuckDbOpt { + return func(d *DuckDb) { + d.ducklakeEnabled = true + d.duckLakeReadOnly = true + d.viewFilters = filters + } } diff --git a/internal/database/duck_db_test.go b/internal/database/duck_db_test.go index 9d7cdbc6..36c15bb2 100644 --- a/internal/database/duck_db_test.go +++ b/internal/database/duck_db_test.go @@ -110,8 +110,8 @@ func Test_executeWithParquetErrorRetry(t *testing.T) { // Helper function to create a test file with proper path structure mkTestFile := func(attempt int) string { - // Create a path that matches the expected format: tp_table=aws_cloudtrail/tp_partition=cloudtrail/tp_date=2024-03-20/test.parquet.N - path := filepath.Join(tmpDir, "tp_table=aws_cloudtrail", "tp_partition=cloudtrail", "tp_date=2024-03-20") + // Create a path that matches the expected format: tp_table=aws_cloudtrail/tp_partition=cloudtrail/year=2024/month=03/test.parquet + path := filepath.Join(tmpDir, "tp_table=aws_cloudtrail", "tp_partition=cloudtrail", "year=2024", "month=03") if err := os.MkdirAll(path, 0755); err != nil { t.Fatalf("failed to create test directory: %v", err) } @@ -206,6 +206,9 @@ func Test_executeWithParquetErrorRetry(t *testing.T) { } func TestDuckDb_WrapperMethods(t *testing.T) { + // TODO fix me + t.Skip("Skipping this test due to CI issues") + // Create a temporary directory for testing tmpDir := t.TempDir() @@ -217,7 +220,9 @@ func TestDuckDb_WrapperMethods(t *testing.T) { // Test Query t.Run("Query", func(t *testing.T) { - rows, err := db.Query("select 1") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + rows, err := db.QueryContext(ctx, "select 1") if err != nil { t.Errorf("Query failed: %v", err) } @@ -228,7 +233,8 @@ func TestDuckDb_WrapperMethods(t *testing.T) { // Test QueryContext t.Run("QueryContext", func(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() rows, err := db.QueryContext(ctx, "select 1") if err != nil { t.Errorf("QueryContext failed: %v", err) @@ -240,7 +246,9 @@ func TestDuckDb_WrapperMethods(t *testing.T) { // Test QueryRow t.Run("QueryRow", func(t *testing.T) { - row := db.QueryRow("select 1") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + row := db.QueryRowContext(ctx, "select 1") if row == nil { t.Error("QueryRow returned nil") } @@ -248,7 +256,8 @@ func TestDuckDb_WrapperMethods(t *testing.T) { // Test QueryRowContext t.Run("QueryRowContext", func(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() row := db.QueryRowContext(ctx, "select 1") if row == nil { t.Error("QueryRowContext returned nil") @@ -257,7 +266,9 @@ func TestDuckDb_WrapperMethods(t *testing.T) { // Test Exec t.Run("Exec", func(t *testing.T) { - result, err := db.Exec("select 1") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + result, err := db.ExecContext(ctx, "select 1") if err != nil { t.Errorf("Exec failed: %v", err) } @@ -268,7 +279,8 @@ func TestDuckDb_WrapperMethods(t *testing.T) { // Test ExecContext t.Run("ExecContext", func(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() result, err := db.ExecContext(ctx, "select 1") if err != nil { t.Errorf("ExecContext failed: %v", err) diff --git a/internal/database/ducklake_table.go b/internal/database/ducklake_table.go new file mode 100644 index 00000000..b774b17c --- /dev/null +++ b/internal/database/ducklake_table.go @@ -0,0 +1,107 @@ +package database + +import ( + "fmt" + "strings" + + "github.com/turbot/tailpipe-plugin-sdk/constants" + "github.com/turbot/tailpipe-plugin-sdk/schema" +) + +// EnsureDuckLakeTable determines whether we have a ducklake table for this table, and if so, whether it needs schema updating +func EnsureDuckLakeTable(columns []*schema.ColumnSchema, db *DuckDb, tableName string) error { + query := fmt.Sprintf("select exists (select 1 from information_schema.tables where table_name = '%s')", tableName) + var exists bool + if err := db.QueryRow(query).Scan(&exists); err != nil { + return err + } + if !exists { + return createDuckLakeTable(columns, db, tableName) + } + return nil +} + +// createDuckLakeTable creates a DuckLake table based on the ConversionSchema +func createDuckLakeTable(columns []*schema.ColumnSchema, db *DuckDb, tableName string) error { + + // Generate the CREATE TABLE SQL + createTableSQL := buildCreateDucklakeTableSQL(columns, tableName) + + // Execute the CREATE TABLE statement + _, err := db.Exec(createTableSQL) + if err != nil { + return fmt.Errorf("failed to create table %s: %w", tableName, err) + } + + // Set partitioning using ALTER TABLE + // partition by the partition, index, year and month + partitionColumns := []string{constants.TpPartition, constants.TpIndex, fmt.Sprintf("year(%s)", constants.TpTimestamp), fmt.Sprintf("month(%s)", constants.TpTimestamp)} + alterTableSQL := fmt.Sprintf(`alter table "%s" set partitioned by (%s);`, + tableName, + strings.Join(partitionColumns, ", ")) + + _, err = db.Exec(alterTableSQL) + if err != nil { + return fmt.Errorf("failed to set partitioning for table %s: %w", tableName, err) + } + + return nil +} + +// buildCreateDucklakeTableSQL generates the CREATE TABLE SQL statement based on the ConversionSchema +func buildCreateDucklakeTableSQL(columns []*schema.ColumnSchema, tableName string) string { + // Build column definitions in sorted order + var columnDefinitions []string + for _, column := range columns { + columnDef := buildColumnDefinition(column) + columnDefinitions = append(columnDefinitions, columnDef) + } + + return fmt.Sprintf(`create table if not exists "%s" ( +%s +);`, + tableName, + strings.Join(columnDefinitions, ",\n")) +} + +// buildColumnDefinition generates the SQL definition for a single column +func buildColumnDefinition(column *schema.ColumnSchema) string { + columnName := fmt.Sprintf("\"%s\"", column.ColumnName) + + // Handle different column types + switch column.Type { + case "struct": + // For struct types, we need to build the struct definition + structDef := buildStructDefinition(column) + return fmt.Sprintf("\t%s %s", columnName, structDef) + case "json": + // json type + return fmt.Sprintf("\t%s json", columnName) + default: + // For scalar types, just use the type directly (lower case) + return fmt.Sprintf("\t%s %s", columnName, strings.ToLower(column.Type)) + } +} + +// buildStructDefinition generates the SQL struct definition for a struct column +func buildStructDefinition(column *schema.ColumnSchema) string { + if len(column.StructFields) == 0 { + return "struct" + } + + var fieldDefinitions []string + for _, field := range column.StructFields { + fieldName := fmt.Sprintf("\"%s\"", field.ColumnName) + fieldType := strings.ToLower(field.Type) + + if field.Type == "struct" { + // Recursively build nested struct definition + nestedStruct := buildStructDefinition(field) + fieldDefinitions = append(fieldDefinitions, fmt.Sprintf("%s %s", fieldName, nestedStruct)) + } else { + fieldDefinitions = append(fieldDefinitions, fmt.Sprintf("%s %s", fieldName, fieldType)) + } + } + + return fmt.Sprintf("struct(%s)", strings.Join(fieldDefinitions, ", ")) +} diff --git a/internal/database/file_metadata.go b/internal/database/file_metadata.go new file mode 100644 index 00000000..6d48665c --- /dev/null +++ b/internal/database/file_metadata.go @@ -0,0 +1,102 @@ +package database + +import ( + "context" + "fmt" + + "github.com/turbot/pipe-fittings/v2/constants" +) + +// FileMetadata represents the result of a file metadata query +type FileMetadata struct { + FileSize int64 + FileCount int64 + RowCount int64 +} + +// TableExists checks if a table exists in the DuckLake metadata tables +func TableExists(ctx context.Context, tableName string, db *DuckDb) (bool, error) { + query := fmt.Sprintf(`select count(*) from %s.ducklake_table where table_name = ?`, constants.DuckLakeMetadataCatalog) + + var count int64 + err := db.QueryRowContext(ctx, query, tableName).Scan(&count) + if err != nil { + return false, fmt.Errorf("unable to check if table %s exists: %w", tableName, err) + } + + return count > 0, nil +} + +// GetTableFileMetadata gets file metadata for a specific table from DuckLake metadata tables +func GetTableFileMetadata(ctx context.Context, tableName string, db *DuckDb) (*FileMetadata, error) { + // first see if the table exists + exists, err := TableExists(ctx, tableName, db) + if err != nil { + return nil, fmt.Errorf("unable to check if table %s exists: %w", tableName, err) + } + if !exists { + // leave everything at zero + return &FileMetadata{}, nil + } + + query := fmt.Sprintf(`select + sum(f.file_size_bytes) as total_size, + count(*) as file_count, + sum(f.record_count) as row_count +from %s.ducklake_data_file f + join %s.ducklake_partition_info p on f.partition_id = p.partition_id + join %s.ducklake_table tp on p.table_id = tp.table_id +where tp.table_name = ? and f.end_snapshot is null`, + constants.DuckLakeMetadataCatalog, + constants.DuckLakeMetadataCatalog, + constants.DuckLakeMetadataCatalog) + + var totalSize, fileCount, rowCount int64 + err = db.QueryRowContext(ctx, query, tableName).Scan(&totalSize, &fileCount, &rowCount) + if err != nil { + return nil, fmt.Errorf("unable to obtain file metadata for table %s: %w", tableName, err) + } + + return &FileMetadata{ + FileSize: totalSize, + FileCount: fileCount, + RowCount: rowCount, + }, nil +} + +// GetPartitionFileMetadata gets file metadata for a specific partition from DuckLake metadata tables +func GetPartitionFileMetadata(ctx context.Context, tableName, partitionName string, db *DuckDb) (*FileMetadata, error) { + // first see if the table exists + exists, err := TableExists(ctx, tableName, db) + if err != nil { + return nil, fmt.Errorf("unable to check if table %s exists: %w", tableName, err) + } + if !exists { + // leave everything at zero + return &FileMetadata{}, nil + } + + query := fmt.Sprintf(`select + coalesce(sum(f.file_size_bytes), 0) as total_size, + coalesce(count(*), 0) as file_count, + coalesce(sum(f.record_count), 0) as row_count +from %s.ducklake_data_file f + join %s.ducklake_file_partition_value fpv on f.data_file_id = fpv.data_file_id + join %s.ducklake_table tp on fpv.table_id = tp.table_id +where tp.table_name = ? and fpv.partition_value = ? and f.end_snapshot is null`, + constants.DuckLakeMetadataCatalog, + constants.DuckLakeMetadataCatalog, + constants.DuckLakeMetadataCatalog) + + var totalSize, fileCount, rowCount int64 + err = db.QueryRowContext(ctx, query, tableName, partitionName).Scan(&totalSize, &fileCount, &rowCount) + if err != nil { + return nil, fmt.Errorf("unable to obtain file metadata for partition %s.%s: %w", tableName, partitionName, err) + } + + return &FileMetadata{ + FileSize: totalSize, + FileCount: fileCount, + RowCount: rowCount, + }, nil +} diff --git a/internal/database/partition_key.go b/internal/database/partition_key.go new file mode 100644 index 00000000..053ad58d --- /dev/null +++ b/internal/database/partition_key.go @@ -0,0 +1,175 @@ +package database + +import ( + "context" + "fmt" + "github.com/turbot/tailpipe/internal/config" + "sort" +) + +// partitionKey is used to uniquely identify a a combination of ducklake partition columns: +// tp_table, tp_partition, tp_index, year(tp_timestamp), month(tp_timestamp) +// It also stores the file and row stats for that partition key +type partitionKey struct { + tpTable string + tpPartition string + tpIndex string + year string // year(tp_timestamp) from partition value + month string // month(tp_timestamp) from partition value + fileCount int // number of files for this partition key + partitionConfig *config.Partition +} + +// query the ducklake_data_file table to get all partition keys combinations which satisfy the provided patterns, +// along with the file and row stats for each partition key combination +func getPartitionKeysMatchingPattern(ctx context.Context, db *DuckDb, patterns []*PartitionPattern) ([]*partitionKey, error) { + // This query joins the DuckLake metadata tables to get partition key combinations: + // - ducklake_data_file: contains file metadata and links to tables + // - ducklake_file_partition_value: contains partition values for each file + // - ducklake_table: contains table names + // + // The partition key structure is: + // - fpv1 (index 0): tp_partition (e.g., "2024-07") + // - fpv2 (index 1): tp_index (e.g., "index1") + // - fpv3 (index 2): year(tp_timestamp) (e.g., "2024") + // - fpv4 (index 3): month(tp_timestamp) (e.g., "7") + // + // We group by these partition keys and count files per combination, + // filtering for active files (end_snapshot is null) + // NOTE: Assumes partitions are defined in order: tp_partition (0), tp_index (1), year(tp_timestamp) (2), month(tp_timestamp) (3) + query := `select + t.table_name as tp_table, + fpv1.partition_value as tp_partition, + fpv2.partition_value as tp_index, + fpv3.partition_value as year, + fpv4.partition_value as month, + count(*) as file_count +from __ducklake_metadata_tailpipe_ducklake.ducklake_data_file df +join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv1 + on df.data_file_id = fpv1.data_file_id and fpv1.partition_key_index = 0 +join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv2 + on df.data_file_id = fpv2.data_file_id and fpv2.partition_key_index = 1 +join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv3 + on df.data_file_id = fpv3.data_file_id and fpv3.partition_key_index = 2 +join __ducklake_metadata_tailpipe_ducklake.ducklake_file_partition_value fpv4 + on df.data_file_id = fpv4.data_file_id and fpv4.partition_key_index = 3 +join __ducklake_metadata_tailpipe_ducklake.ducklake_table t + on df.table_id = t.table_id +where df.end_snapshot is null +group by + t.table_name, + fpv1.partition_value, + fpv2.partition_value, + fpv3.partition_value, + fpv4.partition_value;` + + rows, err := db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to get partition keys requiring compaction: %w", err) + } + defer rows.Close() + + var partitionKeys []*partitionKey + for rows.Next() { + var pk = &partitionKey{} + + if err := rows.Scan(&pk.tpTable, &pk.tpPartition, &pk.tpIndex, &pk.year, &pk.month, &pk.fileCount); err != nil { + return nil, fmt.Errorf("failed to scan partition key row: %w", err) + } + + // retrieve the partition config for this key (which may not exist - that is ok + partitionConfig, ok := config.GlobalConfig.Partitions[pk.partitionName()] + if ok { + pk.partitionConfig = partitionConfig + } + + // check whether this partition key matches any of the provided patterns and whether there are any files + if pk.fileCount > 0 && PartitionMatchesPatterns(pk.tpTable, pk.tpPartition, patterns) { + partitionKeys = append(partitionKeys, pk) + } + } + + return partitionKeys, nil +} + +// findOverlappingFileRanges finds sets of files that have overlapping time ranges and converts them to unorderedDataTimeRange +func (p *partitionKey) findOverlappingFileRanges(fileRanges []fileTimeRange) ([]unorderedDataTimeRange, error) { + if len(fileRanges) <= 1 { + return []unorderedDataTimeRange{}, nil + } + + // Sort by start time - O(n log n) + sort.Slice(fileRanges, func(i, j int) bool { + return fileRanges[i].min.Before(fileRanges[j].min) + }) + + var unorderedRanges []unorderedDataTimeRange + processedFiles := make(map[string]struct{}) + + for i, currentFile := range fileRanges { + if _, processed := processedFiles[currentFile.path]; processed { + continue + } + + // Find all files that overlap with this one + overlappingFiles := p.findFilesOverlappingWith(currentFile, fileRanges[i+1:], processedFiles) + + // Only keep sets with multiple files (single files don't need compaction) + if len(overlappingFiles) > 1 { + // Convert overlapping files to unorderedDataTimeRange + timeRange, err := newUnorderedDataTimeRange(overlappingFiles) + if err != nil { + return nil, fmt.Errorf("failed to create unordered time range: %w", err) + } + unorderedRanges = append(unorderedRanges, timeRange) + } + } + + return unorderedRanges, nil +} + +// findFilesOverlappingWith finds all files that overlap with the given file +func (p *partitionKey) findFilesOverlappingWith(startFile fileTimeRange, remainingFiles []fileTimeRange, processedFiles map[string]struct{}) []fileTimeRange { + overlappingFileRanges := []fileTimeRange{startFile} + processedFiles[startFile.path] = struct{}{} + setMaxEnd := startFile.max + + for _, candidateFile := range remainingFiles { + if _, processed := processedFiles[candidateFile.path]; processed { + continue + } + + // Early termination: if candidate starts after set ends, no more overlaps + if candidateFile.min.After(setMaxEnd) { + break + } + + // Check if this file overlaps with any file in our set + if p.fileOverlapsWithSet(candidateFile, overlappingFileRanges) { + overlappingFileRanges = append(overlappingFileRanges, candidateFile) + processedFiles[candidateFile.path] = struct{}{} + + // Update set's max end time + if candidateFile.max.After(setMaxEnd) { + setMaxEnd = candidateFile.max + } + } + } + + return overlappingFileRanges +} + +// fileOverlapsWithSet checks if a file overlaps with any file in the set +func (p *partitionKey) fileOverlapsWithSet(candidateFile fileTimeRange, fileSet []fileTimeRange) bool { + for _, setFile := range fileSet { + if rangesOverlap(setFile, candidateFile) { + return true + } + } + return false +} + +// return fully qualified partition name (table.partition) +func (p *partitionKey) partitionName() string { + return fmt.Sprintf("%s.%s", p.tpTable, p.tpPartition) +} diff --git a/internal/database/partition_key_test.go b/internal/database/partition_key_test.go new file mode 100644 index 00000000..f995aa55 --- /dev/null +++ b/internal/database/partition_key_test.go @@ -0,0 +1,383 @@ +package database + +import ( + "testing" + "time" +) + +// timeString is a helper function to create time.Time from string +func timeString(timeStr string) time.Time { + t, err := time.Parse("2006-01-02 15:04:05", timeStr) + if err != nil { + panic(err) + } + return t +} + +func TestPartitionKeyRangeOperations(t *testing.T) { + pk := &partitionKey{} + + tests := []struct { + name string + testType string // "rangesOverlap", "findOverlappingFileRanges", "newUnorderedDataTimeRange" + input interface{} + expected interface{} + }{ + // Test cases for rangesOverlap function + { + name: "rangesOverlap - overlapping ranges", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-03 00:00:00")}, + }, + expected: true, + }, + { + name: "rangesOverlap - non-overlapping ranges", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-03 00:00:00"), max: timeString("2024-01-04 00:00:00")}, + }, + expected: false, + }, + { + name: "rangesOverlap - touching ranges (contiguous, not overlapping)", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-02 00:00:00"), max: timeString("2024-01-03 00:00:00")}, + }, + expected: false, + }, + { + name: "rangesOverlap - identical ranges", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00")}, + }, + expected: true, + }, + { + name: "rangesOverlap - partial overlap", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 12:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-02 00:00:00"), max: timeString("2024-01-03 00:00:00")}, + }, + expected: true, + }, + { + name: "rangesOverlap - one range completely inside another", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-05 00:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-02 00:00:00"), max: timeString("2024-01-03 00:00:00")}, + }, + expected: true, + }, + { + name: "rangesOverlap - ranges with same start time", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-03 00:00:00")}, + }, + expected: true, + }, + { + name: "rangesOverlap - ranges with same end time", + testType: "rangesOverlap", + input: struct { + r1 fileTimeRange + r2 fileTimeRange + }{ + r1: fileTimeRange{min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00")}, + r2: fileTimeRange{min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-02 00:00:00")}, + }, + expected: true, + }, + + // Test cases for findOverlappingFileRanges function + { + name: "findOverlappingFileRanges - no overlaps", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-03 00:00:00"), max: timeString("2024-01-04 00:00:00"), rowCount: 2000}, + {path: "file3", min: timeString("2024-01-05 00:00:00"), max: timeString("2024-01-06 00:00:00"), rowCount: 1500}, + }, + expected: []unorderedDataTimeRange{}, + }, + { + name: "findOverlappingFileRanges - simple overlap", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-03 00:00:00"), rowCount: 2000}, + }, + expected: []unorderedDataTimeRange{ + { + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-03 00:00:00"), + RowCount: 3000, + }, + }, + }, + { + name: "findOverlappingFileRanges - cross-overlapping sets", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-03 00:00:00"), rowCount: 2000}, + {path: "file3", min: timeString("2024-01-02 12:00:00"), max: timeString("2024-01-04 00:00:00"), rowCount: 1500}, + {path: "file4", min: timeString("2024-01-03 12:00:00"), max: timeString("2024-01-05 00:00:00"), rowCount: 1800}, + }, + expected: []unorderedDataTimeRange{ + { + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-05 00:00:00"), + RowCount: 6300, + }, + }, + }, + { + name: "findOverlappingFileRanges - multiple separate groups", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-03 00:00:00"), rowCount: 2000}, + {path: "file3", min: timeString("2024-01-05 00:00:00"), max: timeString("2024-01-06 00:00:00"), rowCount: 1500}, + {path: "file4", min: timeString("2024-01-05 12:00:00"), max: timeString("2024-01-07 00:00:00"), rowCount: 1800}, + }, + expected: []unorderedDataTimeRange{ + { + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-03 00:00:00"), + RowCount: 3000, + }, + { + StartTime: timeString("2024-01-05 00:00:00"), + EndTime: timeString("2024-01-07 00:00:00"), + RowCount: 3300, + }, + }, + }, + { + name: "findOverlappingFileRanges - single file", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + }, + expected: []unorderedDataTimeRange{}, + }, + { + name: "findOverlappingFileRanges - empty input", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{}, + expected: []unorderedDataTimeRange{}, + }, + { + name: "findOverlappingFileRanges - three overlapping files", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-02 12:00:00"), rowCount: 2000}, + {path: "file3", min: timeString("2024-01-02 00:00:00"), max: timeString("2024-01-03 00:00:00"), rowCount: 1500}, + }, + expected: []unorderedDataTimeRange{ + { + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-03 00:00:00"), + RowCount: 4500, + }, + }, + }, + { + name: "findOverlappingFileRanges - files with identical time ranges", + testType: "findOverlappingFileRanges", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 2000}, + }, + expected: []unorderedDataTimeRange{ + { + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-02 00:00:00"), + RowCount: 3000, + }, + }, + }, + + // Test cases for newUnorderedDataTimeRange function + { + name: "newUnorderedDataTimeRange - single file", + testType: "newUnorderedDataTimeRange", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + }, + expected: unorderedDataTimeRange{ + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-02 00:00:00"), + RowCount: 1000, + }, + }, + { + name: "newUnorderedDataTimeRange - multiple overlapping files", + testType: "newUnorderedDataTimeRange", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-03 00:00:00"), rowCount: 2000}, + {path: "file3", min: timeString("2024-01-02 00:00:00"), max: timeString("2024-01-04 00:00:00"), rowCount: 1500}, + }, + expected: unorderedDataTimeRange{ + StartTime: timeString("2024-01-01 00:00:00"), // earliest start + EndTime: timeString("2024-01-04 00:00:00"), // latest end + RowCount: 4500, // sum of all row counts + }, + }, + { + name: "newUnorderedDataTimeRange - files with zero row counts", + testType: "newUnorderedDataTimeRange", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 0}, + {path: "file2", min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-03 00:00:00"), rowCount: 1000}, + }, + expected: unorderedDataTimeRange{ + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-03 00:00:00"), + RowCount: 1000, + }, + }, + { + name: "newUnorderedDataTimeRange - files with same start time", + testType: "newUnorderedDataTimeRange", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-03 00:00:00"), rowCount: 2000}, + }, + expected: unorderedDataTimeRange{ + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-03 00:00:00"), + RowCount: 3000, + }, + }, + { + name: "newUnorderedDataTimeRange - files with same end time", + testType: "newUnorderedDataTimeRange", + input: []fileTimeRange{ + {path: "file1", min: timeString("2024-01-01 00:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 1000}, + {path: "file2", min: timeString("2024-01-01 12:00:00"), max: timeString("2024-01-02 00:00:00"), rowCount: 2000}, + }, + expected: unorderedDataTimeRange{ + StartTime: timeString("2024-01-01 00:00:00"), + EndTime: timeString("2024-01-02 00:00:00"), + RowCount: 3000, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + switch tt.testType { + case "rangesOverlap": + input := tt.input.(struct { + r1 fileTimeRange + r2 fileTimeRange + }) + result := rangesOverlap(input.r1, input.r2) + expected := tt.expected.(bool) + if result != expected { + t.Errorf("rangesOverlap() = %v, expected %v", result, expected) + } + + case "findOverlappingFileRanges": + input := tt.input.([]fileTimeRange) + expected := tt.expected.([]unorderedDataTimeRange) + result, err := pk.findOverlappingFileRanges(input) + if err != nil { + t.Fatalf("findOverlappingFileRanges() error = %v", err) + } + if !compareUnorderedRangesets(result, expected) { + t.Errorf("findOverlappingFileRanges() = %v, expected %v", result, expected) + } + + case "newUnorderedDataTimeRange": + input := tt.input.([]fileTimeRange) + expected := tt.expected.(unorderedDataTimeRange) + result, err := newUnorderedDataTimeRange(input) + if err != nil { + t.Fatalf("newUnorderedDataTimeRange() error = %v", err) + } + if !result.StartTime.Equal(expected.StartTime) { + t.Errorf("StartTime = %v, expected %v", result.StartTime, expected.StartTime) + } + if !result.EndTime.Equal(expected.EndTime) { + t.Errorf("EndTime = %v, expected %v", result.EndTime, expected.EndTime) + } + if result.RowCount != expected.RowCount { + t.Errorf("RowCount = %v, expected %v", result.RowCount, expected.RowCount) + } + } + }) + } +} + +// compareUnorderedRangesets compares two slices of unorderedDataTimeRange, ignoring order +func compareUnorderedRangesets(actual []unorderedDataTimeRange, expected []unorderedDataTimeRange) bool { + if len(actual) != len(expected) { + return false + } + + // Convert to sets for comparison using time range as key + actualSets := make(map[string]unorderedDataTimeRange) + expectedSets := make(map[string]unorderedDataTimeRange) + + for _, set := range actual { + key := set.StartTime.Format("2006-01-02 15:04:05") + "-" + set.EndTime.Format("2006-01-02 15:04:05") + actualSets[key] = set + } + + for _, set := range expected { + key := set.StartTime.Format("2006-01-02 15:04:05") + "-" + set.EndTime.Format("2006-01-02 15:04:05") + expectedSets[key] = set + } + + // Check if each set in actual has a matching set in expected + for key, actualSet := range actualSets { + expectedSet, exists := expectedSets[key] + if !exists || !unorderedRangesetsEqual(actualSet, expectedSet) { + return false + } + } + + return true +} + +// unorderedRangesetsEqual compares two unorderedDataTimeRange structs +func unorderedRangesetsEqual(a, b unorderedDataTimeRange) bool { + return a.StartTime.Equal(b.StartTime) && a.EndTime.Equal(b.EndTime) && a.RowCount == b.RowCount +} diff --git a/internal/database/partition_pattern.go b/internal/database/partition_pattern.go new file mode 100644 index 00000000..ddaae33d --- /dev/null +++ b/internal/database/partition_pattern.go @@ -0,0 +1,123 @@ +package database + +import ( + "fmt" + "strings" + + "github.com/danwakefield/fnmatch" + "github.com/turbot/tailpipe/internal/config" + "golang.org/x/exp/maps" +) + +// PartitionPattern represents a pattern used to match partitions. +// It consists of a table pattern and a partition pattern, both of which are +// used to match a given table and partition name. +type PartitionPattern struct { + Table string + Partition string +} + +func NewPartitionPattern(partition *config.Partition) PartitionPattern { + return PartitionPattern{ + Table: partition.TableName, + Partition: partition.ShortName, + } +} + +// PartitionMatchesPatterns checks if the given table and partition match any of the provided patterns. +func PartitionMatchesPatterns(table, partition string, patterns []*PartitionPattern) bool { + if len(patterns) == 0 { + return true + } + // do ANY patterns match + gotMatch := false + for _, pattern := range patterns { + if fnmatch.Match(pattern.Table, table, fnmatch.FNM_CASEFOLD) && + fnmatch.Match(pattern.Partition, partition, fnmatch.FNM_CASEFOLD) { + gotMatch = true + } + } + return gotMatch +} + +// GetPartitionsForArg returns the actual partition names that match the given argument. +// The partitionNames list is needed to determine whether a single-part argument refers to a table or partition. +func GetPartitionsForArg(partitionMap map[string]*config.Partition, arg string) ([]string, error) { + partitionNames := maps.Keys(partitionMap) + partitionPattern, err := GetPartitionPatternsForArgs(partitionNames, arg) + if err != nil { + return nil, err + } + // now match the partition + var res []string + for _, partition := range partitionMap { + if PartitionMatchesPatterns(partition.TableName, partition.ShortName, partitionPattern) { + res = append(res, partition.UnqualifiedName) + } + } + return res, nil +} + +// GetPartitionPatternsForArgs returns the table and partition patterns for the given partition args. +// The partitions list is needed to determine whether single-part arguments refer to tables or partitions. +func GetPartitionPatternsForArgs(partitions []string, partitionArgs ...string) ([]*PartitionPattern, error) { + var res []*PartitionPattern + for _, arg := range partitionArgs { + partitionPattern, err := GetPartitionMatchPatternsForArg(partitions, arg) + if err != nil { + return nil, fmt.Errorf("error processing partition arg '%s': %w", arg, err) + } + + res = append(res, partitionPattern) + } + + return res, nil +} + +// GetPartitionMatchPatternsForArg parses a single partition argument into a PartitionPattern. +// The partitions list is needed to determine whether single-part arguments refer to tables or partitions. +func GetPartitionMatchPatternsForArg(partitions []string, arg string) (*PartitionPattern, error) { + var partitionPattern *PartitionPattern + parts := strings.Split(arg, ".") + switch len(parts) { + case 1: + var err error + partitionPattern, err = getPartitionPatternsForSinglePartName(partitions, arg) + if err != nil { + return nil, err + } + case 2: + // use the args as provided + partitionPattern = &PartitionPattern{Table: parts[0], Partition: parts[1]} + default: + return nil, fmt.Errorf("invalid partition name: %s", arg) + } + return partitionPattern, nil +} + +// getPartitionPatternsForSinglePartName determines whether a single-part argument refers to a table or partition. +// The partitions list is needed to check if the argument matches any existing table names. +// e.g. if the arg is "aws*" and it matches table "aws_cloudtrail_log", it's treated as a table pattern. +func getPartitionPatternsForSinglePartName(partitions []string, arg string) (*PartitionPattern, error) { + var tablePattern, partitionPattern string + // '*' is not valid for a single part arg + if arg == "*" { + return nil, fmt.Errorf("invalid partition name: %s", arg) + } + // check whether there is table with this name + // partitions is a list of Unqualified names, i.e.
. + for _, partition := range partitions { + table := strings.Split(partition, ".")[0] + + // if the arg matches a table name, set table pattern to the arg and partition pattern to * + if fnmatch.Match(arg, table, fnmatch.FNM_CASEFOLD) { + tablePattern = arg + partitionPattern = "*" + return &PartitionPattern{Table: tablePattern, Partition: partitionPattern}, nil + } + } + // so there IS NOT a table with this name - set table pattern to * and user provided partition name + tablePattern = "*" + partitionPattern = arg + return &PartitionPattern{Table: tablePattern, Partition: partitionPattern}, nil +} diff --git a/internal/database/partition_pattern_test.go b/internal/database/partition_pattern_test.go new file mode 100644 index 00000000..81158c1d --- /dev/null +++ b/internal/database/partition_pattern_test.go @@ -0,0 +1,296 @@ +package database + +import ( + "github.com/turbot/pipe-fittings/v2/modconfig" + "github.com/turbot/tailpipe/internal/config" + "reflect" + "sort" + "strings" + "testing" +) + +func Test_getPartition(t *testing.T) { + type args struct { + partitions []string + name string + } + tests := []struct { + name string + args args + want []string + wantErr bool + }{ + { + name: "Invalid partition name", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "*", + }, + wantErr: true, + }, + { + name: "Full partition name, exists", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "aws_s3_cloudtrail_log.p1", + }, + want: []string{"aws_s3_cloudtrail_log.p1"}, + }, + { + name: "Full partition name, does not exist", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "aws_s3_cloudtrail_log.p3", + }, + want: nil, + }, + { + name: "Table name", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "aws_s3_cloudtrail_log", + }, + want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + }, + { + name: "Table name (exists) with wildcard", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "aws_s3_cloudtrail_log.*", + }, + want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + }, + { + name: "Table name (exists) with ?", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "aws_s3_cloudtrail_log.p?", + }, + want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + }, + { + name: "Table name (exists) with non matching partition wildacard", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "aws_s3_cloudtrail_log.d*?", + }, + want: nil, + }, + { + name: "Table name (does not exist)) with wildcard", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + name: "foo.*", + }, + want: nil, + }, + { + name: "Partition short name, exists", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, + name: "p1", + }, + want: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, + }, + { + name: "Table wildcard, partition short name, exists", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, + name: "*.p1", + }, + want: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, + }, + { + name: "Partition short name, does not exist", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, + name: "p3", + }, + want: nil, + }, + { + name: "Table wildcard, partition short name, does not exist", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, + name: "*.p3", + }, + want: nil, + }, + { + name: "Table wildcard, no dot", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, + name: "aws*", + }, + want: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1", "aws_elb_access_log.p2"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var partitions = getPartitions(tt.args.partitions) + + got, err := GetPartitionsForArg(partitions, tt.args.name) + if (err != nil) != tt.wantErr { + t.Errorf("getPartitions() error = %v, wantErr %v", err, tt.wantErr) + return + } + // sort the slices before comparing + sort.Strings(tt.want) + sort.Strings(got) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getPartitions() got = %v, want %v", got, tt.want) + } + }) + } +} + +func getPartitions(partitions []string) map[string]*config.Partition { + var partitionMap = make(map[string]*config.Partition) + for _, p := range partitions { + parts := strings.SplitN(p, ".", 2) + if len(parts) != 2 { + continue + } + partitionMap[p] = &config.Partition{ + HclResourceImpl: modconfig.HclResourceImpl{ + UnqualifiedName: p, + ShortName: parts[1], + }, + TableName: parts[0], + } + } + return partitionMap +} + +func Test_getPartitionMatchPatternsForArg(t *testing.T) { + type args struct { + partitions []string + arg string + } + tests := []struct { + name string + args args + wantTablePattern string + wantPartPattern string + wantErr bool + }{ + { + name: "Valid table and partition pattern", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + arg: "aws_s3_cloudtrail_log.p1", + }, + wantTablePattern: "aws_s3_cloudtrail_log", + wantPartPattern: "p1", + }, + { + name: "Wildcard partition pattern", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1"}, + arg: "aws_s3_cloudtrail_log.*", + }, + wantTablePattern: "aws_s3_cloudtrail_log", + wantPartPattern: "*", + }, + { + name: "Wildcard in table and partition both", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2", "aws_elb_access_log.p1"}, + arg: "aws*.*", + }, + wantTablePattern: "aws*", + wantPartPattern: "*", + }, + { + name: "Wildcard table pattern", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, + arg: "*.p1", + }, + wantTablePattern: "*", + wantPartPattern: "p1", + }, + { + name: "Invalid partition name", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + arg: "*", + }, + wantErr: true, + }, + { + name: "Table exists without partition", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_s3_cloudtrail_log.p2"}, + arg: "aws_s3_cloudtrail_log", + }, + wantTablePattern: "aws_s3_cloudtrail_log", + wantPartPattern: "*", + }, + { + name: "Partition only, multiple tables", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1", "aws_elb_access_log.p1"}, + arg: "p1", + }, + wantTablePattern: "*", + wantPartPattern: "p1", + }, + { + name: "Invalid argument with multiple dots", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1"}, + arg: "aws.s3.cloudtrail", + }, + wantErr: true, + }, + { + name: "Non-existing table name", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1"}, + arg: "non_existing_table.p1", + }, + wantTablePattern: "non_existing_table", + wantPartPattern: "p1", + }, + { + name: "Partition name does not exist", + args: args{ + partitions: []string{"aws_s3_cloudtrail_log.p1"}, + arg: "p2", + }, + wantTablePattern: "*", + wantPartPattern: "p2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + partitionPattern, err := GetPartitionMatchPatternsForArg(tt.args.partitions, tt.args.arg) + + if err != nil { + if !tt.wantErr { + t.Errorf("GetPartitionMatchPatternsForArg() error = %v, wantErr %v", err, tt.wantErr) + } + return + } else if tt.wantErr { + t.Errorf("GetPartitionMatchPatternsForArg() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil { + // must be a wanted err + return + } + + gotTablePattern := partitionPattern.Table + gotPartPattern := partitionPattern.Partition + if gotTablePattern != tt.wantTablePattern { + t.Errorf("GetPartitionMatchPatternsForArg() gotTablePattern = %v, want %v", gotTablePattern, tt.wantTablePattern) + } + if gotPartPattern != tt.wantPartPattern { + t.Errorf("GetPartitionMatchPatternsForArg() gotPartPattern = %v, want %v", gotPartPattern, tt.wantPartPattern) + } + }) + } +} diff --git a/internal/database/partitions.go b/internal/database/partitions.go deleted file mode 100644 index 9d6f25c4..00000000 --- a/internal/database/partitions.go +++ /dev/null @@ -1,52 +0,0 @@ -package database - -import ( - "context" - "fmt" - - "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/filepaths" -) - -// ListPartitions uses DuckDB to build a list of all partitions for all tables -func ListPartitions(ctx context.Context) ([]string, error) { - // Hive format is table, partition, index, date - - dataDir := config.GlobalWorkspaceProfile.GetDataDir() - if dataDir == "" { - return nil, fmt.Errorf("data directory is not set") - } - // TODO KAI handle no partitions - - // Build DuckDB query to get the names of all partitions underneath data dir - parquetPath := filepaths.GetParquetFileGlobForTable(dataDir, "*", "") - query := `select distinct tp_table || '.' || tp_partition from read_parquet('` + parquetPath + `', hive_partitioning=true)` - - // Open DuckDB in-memory database - db, err := NewDuckDb() - if err != nil { - return nil, fmt.Errorf("failed to open DuckDB: %v", err) - } - defer db.Close() - - rows, err := db.QueryContext(ctx, query) - if err != nil { - return nil, fmt.Errorf("failed to execute query: %v", err) - } - defer rows.Close() - - var partitions []string - for rows.Next() { - var partition string - if err := rows.Scan(&partition); err != nil { - return nil, fmt.Errorf("failed to scan row: %v", err) - } - partitions = append(partitions, partition) - } - - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error iterating rows: %v", err) - } - - return partitions, nil -} diff --git a/internal/parquet/conversion_helpers.go b/internal/database/read_json_query.go similarity index 53% rename from internal/parquet/conversion_helpers.go rename to internal/database/read_json_query.go index a26fd154..0ff2e1cc 100644 --- a/internal/parquet/conversion_helpers.go +++ b/internal/database/read_json_query.go @@ -1,47 +1,63 @@ -package parquet +package database import ( "fmt" + "log/slog" "strings" - "github.com/turbot/tailpipe-plugin-sdk/constants" - "github.com/turbot/go-kit/helpers" + "github.com/turbot/tailpipe-plugin-sdk/constants" "github.com/turbot/tailpipe-plugin-sdk/schema" + "github.com/turbot/tailpipe/internal/config" ) -// TODO: review this function & add comments: https://github.com/turbot/tailpipe/issues/305 -func buildViewQuery(tableSchema *schema.ConversionSchema) string { - var tpIndexMapped, tpTimestampMapped bool +// buildReadJsonQueryFormat creates a SQL query template for reading JSONL files with DuckDB. +// +// Returns a format string with a %s placeholder for the JSON filename that gets filled in when executed. +// The query is built by constructing a select clause for each field in the conversion schema, +// adding tp_index from partition config, and applying any partition filters (e.g. date filer) +// +// Example output: +// +// select "user_id" as "user_id", "name" as "user_name", "created_at" as "tp_timestamp", +// "default" as "tp_index" +// from read_ndjson(%s, columns = {"user_id": 'varchar', "name": 'varchar', "created_at": 'timestamp'}) +func buildReadJsonQueryFormat(conversionSchema *schema.ConversionSchema, partition *config.Partition) string { + var tpTimestampMapped bool // first build the select clauses - use the table def columns - var selectClauses []string - for _, column := range tableSchema.Columns { - + var selectClauses = []string{ + "row_number() over () as row_id", // add a row_id column to use with validation + } + for _, column := range conversionSchema.Columns { var selectClause string switch column.ColumnName { case constants.TpDate: // skip this column - it is derived from tp_timestamp continue case constants.TpIndex: - tpIndexMapped = true - selectClause = fmt.Sprintf("\tcoalesce(\"%s\", '%s') as \"%s\"", column.SourceName, schema.DefaultIndex, column.ColumnName) + // NOTE: we ignore tp_index in the source data and ONLY add it based ont he default or configured value + slog.Warn("tp_index is a reserved column name and should not be used in the source data. It will be added automatically based on the configured value.") + // skip this column - it will be populated manually using the partition config + continue case constants.TpTimestamp: tpTimestampMapped = true // fallthrough to populate the select clasue as normal fallthrough default: - selectClause = getSelectSqlForField(column, 1) + selectClause = getSelectSqlForField(column) } selectClauses = append(selectClauses, selectClause) } - // default tpIndex to 'default' - // (note - if tpIndex IS mapped, getSelectSqlForField will have added a coalesce statement to default to 'default' - if !tpIndexMapped { - selectClauses = append(selectClauses, fmt.Sprintf("'%s' as tp_index", schema.DefaultIndex)) - } + + // add the tp_index - this is determined by the partition - it defaults to "default" but may be overridden in the partition config + // NOTE: we DO NOT wrap the tp_index expression in quotes - that will have already been done as part of partition config validation + selectClauses = append(selectClauses, fmt.Sprintf("\t%s as \"tp_index\"", partition.TpIndexColumn)) + // if we have a mapping for tp_timestamp, add tp_date as well + // (if we DO NOT have tp_timestamp, the validation will fail - but we want the validation error - + // NOT an error when we try to select tp_date using tp_timestamp as source) if tpTimestampMapped { // Add tp_date after tp_timestamp is defined selectClauses = append(selectClauses, ` case @@ -50,15 +66,24 @@ func buildViewQuery(tableSchema *schema.ConversionSchema) string { } // build column definitions - these will be passed to the read_json function - columnDefinitions := getReadJSONColumnDefinitions(tableSchema.SourceColumns) + columnDefinitions := getReadJSONColumnDefinitions(conversionSchema.SourceColumns) + + var whereClause string + if partition.Filter != "" { + // we need to escape the % in the filter, as it is passed to the fmt.Sprintf function + filter := strings.ReplaceAll(partition.Filter, "%", "%%") + whereClause = fmt.Sprintf("\nwhere %s", filter) + } - return fmt.Sprintf(`select + res := fmt.Sprintf(`select %s from read_ndjson( - '%%s', + %%s, %s - )`, strings.Join(selectClauses, ",\n"), helpers.Tabify(columnDefinitions, "\t")) + )%s`, strings.Join(selectClauses, ",\n"), helpers.Tabify(columnDefinitions, "\t"), whereClause) + + return res } // return the column definitions for the row conversionSchema, in the format required for the duck db read_json_auto function @@ -76,16 +101,17 @@ func getReadJSONColumnDefinitions(sourceColumns []schema.SourceColumnDef) string return str.String() } -// Return the SQL line to select the given field -func getSelectSqlForField(column *schema.ColumnSchema, tabs int) string { - // Calculate the tab spacing - tab := strings.Repeat("\t", tabs) +// getSelectSqlForField builds a SELECT clause for a single field based on its schema definition. +// - If the field has a transform defined, it uses that transform expression. +// - For struct fields, it creates a struct_pack expression to properly construct the nested structure from the source JSON data. +// - All other field types are handled with simple column references. +func getSelectSqlForField(column *schema.ColumnSchema) string { // If the column has a transform, use it if column.Transform != "" { // as this is going into a string format, we need to escape % escapedTransform := strings.ReplaceAll(column.Transform, "%", "%%") - return fmt.Sprintf("%s%s as \"%s\"", tab, escapedTransform, column.ColumnName) + return fmt.Sprintf("\t%s as \"%s\"", escapedTransform, column.ColumnName) } // NOTE: we will have normalised column types to lower case @@ -104,21 +130,16 @@ func getSelectSqlForField(column *schema.ColumnSchema, tabs int) string { str.WriteString(",\n") } parentName := fmt.Sprintf("\"%s\"", column.SourceName) - str.WriteString(getTypeSqlForStructField(nestedColumn, parentName, tabs+2)) + str.WriteString(getTypeSqlForStructField(nestedColumn, parentName, 3)) } // Close struct_pack and case - str.WriteString(fmt.Sprintf("\n%s\t)\n", tab)) - str.WriteString(fmt.Sprintf("%send as \"%s\"", tab, column.ColumnName)) + str.WriteString("\n\t\t)\n") + str.WriteString(fmt.Sprintf("\tend as \"%s\"", column.ColumnName)) return str.String() - - case "json": - // Convert the value using json() - return fmt.Sprintf("%sjson(\"%s\") as \"%s\"", tab, column.SourceName, column.ColumnName) - default: // Scalar fields - return fmt.Sprintf("%s\"%s\" as \"%s\"", tab, column.SourceName, column.ColumnName) + return fmt.Sprintf("\t\"%s\" as \"%s\"", column.SourceName, column.ColumnName) } } diff --git a/internal/parquet/conversion_helpers_test.go b/internal/database/read_json_query_test.go similarity index 99% rename from internal/parquet/conversion_helpers_test.go rename to internal/database/read_json_query_test.go index 420b924a..f70d2a96 100644 --- a/internal/parquet/conversion_helpers_test.go +++ b/internal/database/read_json_query_test.go @@ -1,4 +1,4 @@ -package parquet +package database import ( _ "github.com/marcboeker/go-duckdb/v2" diff --git a/internal/database/reorder_metadata.go b/internal/database/reorder_metadata.go new file mode 100644 index 00000000..67139a50 --- /dev/null +++ b/internal/database/reorder_metadata.go @@ -0,0 +1,40 @@ +package database + +import ( + "context" + "fmt" + "time" +) + +type reorderMetadata struct { + pk *partitionKey + unorderedRanges []unorderedDataTimeRange + + rowCount int64 + maxRowId int64 + minTimestamp time.Time + maxTimestamp time.Time +} + +func newReorderMetadata(ctx context.Context, db *DuckDb, p *partitionKey) (*reorderMetadata, error) { + var rm = &reorderMetadata{pk: p} + + // Query to get row count and time range for this partition + countQuery := fmt.Sprintf(`select count(*), max(rowid), min(tp_timestamp), max(tp_timestamp) from "%s" + where tp_partition = ? + and tp_index = ? + and year(tp_timestamp) = ? + and month(tp_timestamp) = ?`, + p.tpTable) + + err := db.QueryRowContext(ctx, countQuery, + p.tpPartition, + p.tpIndex, + p.year, + p.month).Scan(&rm.rowCount, &rm.maxRowId, &rm.minTimestamp, &rm.maxTimestamp) + if err != nil { + return nil, fmt.Errorf("failed to get row count and time range for partition: %w", err) + } + + return rm, nil +} diff --git a/internal/database/row_validation_error.go b/internal/database/row_validation_error.go new file mode 100644 index 00000000..e76b0cd7 --- /dev/null +++ b/internal/database/row_validation_error.go @@ -0,0 +1,30 @@ +package database + +import ( + "fmt" + "strings" + + "github.com/turbot/pipe-fittings/v2/utils" +) + +type RowValidationError struct { + nullColumns []string + failedRows int64 +} + +func NewRowValidationError(failedRows int64, nullColumns []string) *RowValidationError { + return &RowValidationError{ + nullColumns: nullColumns, + failedRows: failedRows, + } +} + +func (e *RowValidationError) Error() string { + return fmt.Sprintf("%d %s failed validation - found null values in %d %s: %s", e.failedRows, utils.Pluralize("row", int(e.failedRows)), len(e.nullColumns), utils.Pluralize("column", len(e.nullColumns)), strings.Join(e.nullColumns, ", ")) +} + +// Is implements the errors.Is interface to support error comparison +func (e *RowValidationError) Is(target error) bool { + _, ok := target.(*RowValidationError) + return ok +} diff --git a/internal/database/schema_change_error.go b/internal/database/schema_change_error.go new file mode 100644 index 00000000..e4a3c965 --- /dev/null +++ b/internal/database/schema_change_error.go @@ -0,0 +1,24 @@ +package database + +import ( + "fmt" + "strings" +) + +type ColumnSchemaChange struct { + Name string + OldType string + NewType string +} + +type SchemaChangeError struct { + ChangedColumns []ColumnSchemaChange +} + +func (e *SchemaChangeError) Error() string { + changeStrings := make([]string, len(e.ChangedColumns)) + for i, change := range e.ChangedColumns { + changeStrings[i] = fmt.Sprintf("'%s': '%s' -> '%s'", change.Name, change.OldType, change.NewType) + } + return fmt.Sprintf("inferred schema change detected - consider specifying a column type in table definition: %s", strings.Join(changeStrings, ", ")) +} diff --git a/internal/database/schema_comparison.go b/internal/database/schema_comparison.go new file mode 100644 index 00000000..8603317b --- /dev/null +++ b/internal/database/schema_comparison.go @@ -0,0 +1,67 @@ +package database + +import ( + "fmt" + "github.com/turbot/tailpipe-plugin-sdk/schema" + "strings" +) + +// TableSchemaStatus represents the status of a table schema comparison +// this is not used at present but will be used when we implement ducklake schema evolution handling +// It indicates whether the table exists, whether the schema matches, whether it can be migrated by ducklake +type TableSchemaStatus struct { + TableExists bool + SchemaMatches bool + CanMigrate bool + SchemaDiff string +} + +// NewTableSchemaStatusFromComparison compares an existing schema with a conversion schema +// and returns a TableSchemaStatus indicating whether they match, can be migrated, and the differences +func NewTableSchemaStatusFromComparison(existingSchema map[string]schema.ColumnSchema, conversionSchema schema.ConversionSchema) TableSchemaStatus { + var diffParts []string + canMigrate := true + + // Create map of new schema for quick lookup + newSchemaMap := make(map[string]*schema.ColumnSchema) + for _, column := range conversionSchema.Columns { + newSchemaMap[column.ColumnName] = column + } + + // Check for removed columns + for existingColName := range existingSchema { + if _, exists := newSchemaMap[existingColName]; !exists { + diffParts = append(diffParts, fmt.Sprintf("- column %s removed", existingColName)) + canMigrate = false + } + } + + // Check for new/modified columns + hasNewColumns := false + for _, column := range conversionSchema.Columns { + existingCol, ok := existingSchema[column.ColumnName] + if !ok { + diffParts = append(diffParts, fmt.Sprintf("+ column %s added (%s)", column.ColumnName, column.Type)) + hasNewColumns = true + continue + } + + if existingCol.Type != column.Type { + diffParts = append(diffParts, fmt.Sprintf("~ column %s type changed: %s ā†’ %s", + column.ColumnName, existingCol.Type, column.Type)) + canMigrate = false + } + } + + matches := len(diffParts) == 0 + if !matches && canMigrate { + canMigrate = hasNewColumns // Only true if we only have additive changes + } + + return TableSchemaStatus{ + TableExists: true, + SchemaMatches: matches, + CanMigrate: canMigrate, + SchemaDiff: strings.Join(diffParts, "\n"), + } +} diff --git a/internal/database/sql_command.go b/internal/database/sql_command.go new file mode 100644 index 00000000..beec0777 --- /dev/null +++ b/internal/database/sql_command.go @@ -0,0 +1,7 @@ +package database + +// SqlCommand represents a SQL command with its description. +type SqlCommand struct { + Description string + Command string +} diff --git a/internal/database/tables.go b/internal/database/tables.go index b8f8b97c..a38c1485 100644 --- a/internal/database/tables.go +++ b/internal/database/tables.go @@ -3,201 +3,71 @@ package database import ( "context" "fmt" - "log/slog" - "os" - "regexp" - "slices" "strings" - "github.com/turbot/pipe-fittings/v2/error_helpers" - "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/filepaths" + "github.com/turbot/pipe-fittings/v2/constants" + "github.com/turbot/tailpipe-plugin-sdk/schema" ) -// AddTableViews creates a view for each table in the data directory, applying the provided duck db filters to the view query -func AddTableViews(ctx context.Context, db *DuckDb, filters ...string) error { - tables, err := getDirNames(config.GlobalWorkspaceProfile.GetDataDir()) +// GetTables returns the list of tables in the DuckLake metadata catalog +func GetTables(ctx context.Context, db *DuckDb) ([]string, error) { + + query := fmt.Sprintf("select table_name from %s.ducklake_table", constants.DuckLakeMetadataCatalog) + rows, err := db.QueryContext(ctx, query) if err != nil { - return fmt.Errorf("failed to get tables: %w", err) + return nil, fmt.Errorf("failed to get tables: %w", err) } + defer rows.Close() - // optimisation - it seems the first time DuckDB creates a view which inspects the file system it is slow - // creating and empty view first and then dropping it seems to speed up the process - createAndDropEmptyView(ctx, db) - - //create a view for each table - for _, tableFolder := range tables { - // create a view for the table - // the tab;le folder is a hive partition folder so will have the format tp_table=table_name - table := strings.TrimPrefix(tableFolder, "tp_table=") - err = AddTableView(ctx, table, db, filters...) + var tableViews []string + for rows.Next() { + var tableView string + err = rows.Scan(&tableView) if err != nil { - return err + return nil, fmt.Errorf("failed to scan table view: %w", err) } + tableViews = append(tableViews, tableView) } - return nil -} - -// NOTE: tactical optimisation - it seems the first time DuckDB creates a view which inspects the file system it is slow -// creating and empty view first and then dropping it seems to speed up the process -func createAndDropEmptyView(ctx context.Context, db *DuckDb) { - _ = AddTableView(ctx, "empty", db) - // drop again - _, _ = db.ExecContext(ctx, "DROP VIEW empty") + return tableViews, nil } -func AddTableView(ctx context.Context, tableName string, db *DuckDb, filters ...string) error { - slog.Info("creating view", "table", tableName, "filters", filters) - - dataDir := config.GlobalWorkspaceProfile.GetDataDir() - // Path to the Parquet directory - // hive structure is /tp_table=/tp_partition=/tp_index=/tp_date=.parquet - parquetPath := filepaths.GetParquetFileGlobForTable(dataDir, tableName, "") +// GetTableSchema returns the schema of the specified table as a map of column names to their types +func GetTableSchema(ctx context.Context, tableName string, db *DuckDb) (map[string]string, error) { + query := fmt.Sprintf(`select c.column_name, c.column_type +from %s.ducklake_table t +join %s.ducklake_column c + on t.table_id = c.table_id +where t.table_name = ? +order by c.column_name;`, constants.DuckLakeMetadataCatalog, constants.DuckLakeMetadataCatalog) - // Step 1: Query the first Parquet file to infer columns - columns, err := getColumnNames(ctx, parquetPath, db) + rows, err := db.QueryContext(ctx, query, tableName) if err != nil { - // if this is because no parquet files match, suppress the error - if strings.Contains(err.Error(), "IO Error: No files found that match the pattern") || error_helpers.IsCancelledError(err) { - return nil - } - return err - } - - // Step 2: Build the select clause - cast tp_index as string - // (this is necessary as duckdb infers the type from the partition column name - // if the index looks like a number, it will infer the column as an int) - var typeOverrides = map[string]string{ - "tp_partition": "varchar", - "tp_index": "varchar", - "tp_date": "date", - } - var selectClauses []string - for _, col := range columns { - wrappedCol := fmt.Sprintf(`"%s"`, col) - if overrideType, ok := typeOverrides[col]; ok { - // Apply the override with casting - selectClauses = append(selectClauses, fmt.Sprintf("cast(%s as %s) as %s", col, overrideType, wrappedCol)) - } else { - // Add the column as-is - selectClauses = append(selectClauses, wrappedCol) - } - } - selectClause := strings.Join(selectClauses, ", ") - - // Step 3: Build the where clause - filterString := "" - if len(filters) > 0 { - filterString = fmt.Sprintf(" where %s", strings.Join(filters, " and ")) - } - - // Step 4: Construct the final query - query := fmt.Sprintf( - "create or replace view %s as select %s from '%s'%s", - tableName, selectClause, parquetPath, filterString, - ) - - // Execute the query - _, err = db.ExecContext(ctx, query) - if err != nil { - slog.Warn("failed to create view", "table", tableName, "error", err) - return fmt.Errorf("failed to create view: %w", err) - } - slog.Info("created view", "table", tableName) - return nil -} - -// query the provided parquet path to get the columns -func getColumnNames(ctx context.Context, parquetPath string, db *DuckDb) ([]string, error) { - columnQuery := fmt.Sprintf("select * from '%s' limit 0", parquetPath) - rows, err := db.QueryContext(ctx, columnQuery) - if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get view schema for %s: %w", tableName, err) } defer rows.Close() - // Retrieve column names - columns, err := rows.Columns() - if err != nil { - return nil, err - } - - // Sort column names alphabetically but with tp_ fields on the end - tpPrefix := "tp_" - slices.SortFunc(columns, func(a, b string) int { - isPrefixedA, isPrefixedB := strings.HasPrefix(a, tpPrefix), strings.HasPrefix(b, tpPrefix) - switch { - case isPrefixedA && !isPrefixedB: - return 1 // a > b - case !isPrefixedA && isPrefixedB: - return -1 // a < b - default: - return strings.Compare(a, b) // standard alphabetical sort + schema := make(map[string]string) + for rows.Next() { + var columnName, columnType string + err = rows.Scan(&columnName, &columnType) + if err != nil { + return nil, fmt.Errorf("failed to scan column schema: %w", err) } - }) - - return columns, nil -} - -func getDirNames(folderPath string) ([]string, error) { - var dirNames []string - - // Read the directory contents - files, err := os.ReadDir(folderPath) - if err != nil { - return nil, err - } - - // Loop through the contents and add directories to dirNames - for _, file := range files { - if file.IsDir() { - dirNames = append(dirNames, file.Name()) + if strings.HasPrefix(columnType, "struct") { + columnType = "struct" } + schema[columnName] = columnType } - return dirNames, nil -} - -func GetRowCount(ctx context.Context, tableName string, partitionName *string) (int64, error) { - // Open a DuckDB connection - db, err := NewDuckDb(WithDbFile(filepaths.TailpipeDbFilePath())) - if err != nil { - return 0, fmt.Errorf("failed to open DuckDB connection: %w", err) - } - defer db.Close() - - var tableNameRegex = regexp.MustCompile(`^[a-zA-Z0-9_]+$`) - if !tableNameRegex.MatchString(tableName) { - return 0, fmt.Errorf("invalid table name") - } - query := fmt.Sprintf("select count(*) from %s", tableName) // #nosec G201 // this is a controlled query tableName must match a regex - if partitionName != nil { - query = fmt.Sprintf("select count(*) from %s where tp_partition = '%s'", tableName, *partitionName) // #nosec G201 // this is a controlled query tableName must match a regex - } - rows, err := db.QueryContext(ctx, query) - if err != nil { - return 0, fmt.Errorf("failed to get row count: %w", err) + if err = rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating over view schema rows: %w", err) } - defer rows.Close() - var count int64 - if rows.Next() { - err = rows.Scan(&count) - if err != nil { - return 0, fmt.Errorf("failed to scan row count: %w", err) - } - } - return count, nil + return schema, nil } -func GetTableViews(ctx context.Context) ([]string, error) { - // Open a DuckDB connection - db, err := NewDuckDb(WithDbFile(filepaths.TailpipeDbFilePath())) - if err != nil { - return nil, fmt.Errorf("failed to open DuckDB connection: %w", err) - } - defer db.Close() - +// GetLegacyTableViews retrieves the names of all table views in the legacy database(tailpipe.db) file +func GetLegacyTableViews(ctx context.Context, db *DuckDb) ([]string, error) { query := "select table_name from information_schema.tables where table_type='VIEW';" rows, err := db.QueryContext(ctx, query) if err != nil { @@ -217,14 +87,8 @@ func GetTableViews(ctx context.Context) ([]string, error) { return tableViews, nil } -func GetTableViewSchema(ctx context.Context, viewName string) (map[string]string, error) { - // Open a DuckDB connection - db, err := NewDuckDb(WithDbFile(filepaths.TailpipeDbFilePath())) - if err != nil { - return nil, fmt.Errorf("failed to open DuckDB connection: %w", err) - } - defer db.Close() - +// GetLegacyTableViewSchema retrieves the schema of a table view in the legacy database(tailpipe.db) file +func GetLegacyTableViewSchema(ctx context.Context, viewName string, db *DuckDb) (*schema.TableSchema, error) { query := ` select column_name, data_type from information_schema.columns @@ -236,22 +100,159 @@ func GetTableViewSchema(ctx context.Context, viewName string) (map[string]string } defer rows.Close() - schema := make(map[string]string) + ts := &schema.TableSchema{ + Name: viewName, + Columns: []*schema.ColumnSchema{}, + } for rows.Next() { + // here each row is a column, so we need to populate the TableSchema.Columns, particularly the + // ColumnName, Type and StructFields var columnName, columnType string err = rows.Scan(&columnName, &columnType) if err != nil { return nil, fmt.Errorf("failed to scan column schema: %w", err) } - if strings.HasPrefix(columnType, "struct") { - columnType = "struct" + + // NOTE: legacy tailpipe views may include `rowid` which we must exclude from the schema as this is a DuckDb system column + // that is automatically added to every table + if columnName == "rowid" { + continue } - schema[columnName] = columnType + col := buildColumnSchema(columnName, columnType) + ts.Columns = append(ts.Columns, col) } if err = rows.Err(); err != nil { return nil, fmt.Errorf("error iterating over view schema rows: %w", err) } - return schema, nil + return ts, nil +} + +// buildColumnSchema constructs a ColumnSchema from a DuckDB data type string. +// It handles primitive types as well as struct and struct[] recursively, populating StructFields. +func buildColumnSchema(columnName string, duckdbType string) *schema.ColumnSchema { + t := strings.TrimSpace(duckdbType) + lower := strings.ToLower(t) + + // Helper to set basic column properties + newCol := func(name, typ string, children []*schema.ColumnSchema) *schema.ColumnSchema { + return &schema.ColumnSchema{ + ColumnName: name, + SourceName: name, + Type: typ, + StructFields: children, + } + } + + // Detect struct or struct[] + if strings.HasPrefix(lower, "struct(") || strings.HasPrefix(lower, "struct ") { + isArray := false + // Handle optional [] suffix indicating array of struct + if strings.HasSuffix(lower, ")[]") { + isArray = true + } + // Extract inner content between the first '(' and the matching ')' + start := strings.Index(t, "(") + end := strings.LastIndex(t, ")") + inner := "" + if start >= 0 && end > start { + inner = strings.TrimSpace(t[start+1 : end]) + } + + fields := parseStructFields(inner) + typ := "struct" + if isArray { + typ = "struct[]" + } + return newCol(columnName, typ, fields) + } + + // Primitive or other complex types - just set as-is + return newCol(columnName, lower, nil) +} + +// parseStructFields parses the content inside a DuckDB struct(...) definition into ColumnSchemas. +// It supports nested struct/struct[] types by recursively building ColumnSchemas for child fields. +func parseStructFields(inner string) []*schema.ColumnSchema { + // Split by top-level commas (not within nested parentheses) + parts := splitTopLevel(inner, ',') + var fields []*schema.ColumnSchema + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + // parse field name (optionally quoted) and type + name, typ := parseFieldNameAndType(p) + if name == "" || typ == "" { + continue + } + col := buildColumnSchema(name, typ) + fields = append(fields, col) + } + return fields +} + +// parseFieldNameAndType parses a single struct field spec of the form: +// +// name type +// "name with spaces" type +// +// where type may itself be struct(...)[]. Returns name and the raw type string. +func parseFieldNameAndType(s string) (string, string) { + s = strings.TrimSpace(s) + if s == "" { + return "", "" + } + if s[0] == '"' { + // quoted name + // find closing quote + i := 1 + for i < len(s) && s[i] != '"' { + i++ + } + if i >= len(s) { + return "", "" + } + name := s[1:i] + rest := strings.TrimSpace(s[i+1:]) + // rest should start with the type + return name, rest + } + // unquoted name up to first space + idx := strings.IndexFunc(s, func(r rune) bool { return r == ' ' || r == '\t' }) + if idx == -1 { + // no type specified + return "", "" + } + name := strings.TrimSpace(s[:idx]) + typ := strings.TrimSpace(s[idx+1:]) + return name, typ +} + +// splitTopLevel splits s by sep, ignoring separators enclosed in parentheses. +func splitTopLevel(s string, sep rune) []string { + var res []string + level := 0 + start := 0 + for i, r := range s { + switch r { + case '(': + level++ + case ')': + if level > 0 { + level-- + } + } + if r == sep && level == 0 { + res = append(res, strings.TrimSpace(s[start:i])) + start = i + 1 + } + } + // add last segment + if start <= len(s) { + res = append(res, strings.TrimSpace(s[start:])) + } + return res } diff --git a/internal/database/views.go b/internal/database/views.go new file mode 100644 index 00000000..ab9e5890 --- /dev/null +++ b/internal/database/views.go @@ -0,0 +1,34 @@ +package database + +import ( + "context" + "fmt" + "strings" + + pconstants "github.com/turbot/pipe-fittings/v2/constants" +) + +// GetCreateViewsSql returns the SQL commands to create views for all tables in the DuckLake catalog, +// +// applying the specified filters. +func GetCreateViewsSql(ctx context.Context, db *DuckDb, filters ...string) ([]SqlCommand, error) { + // get list of tables + tables, err := GetTables(ctx, db) + if err != nil { + return nil, fmt.Errorf("failed to get db tables: %w", err) + } + + // Step 3: Build the where clause + filterString := "" + if len(filters) > 0 { + filterString = fmt.Sprintf(" where %s", strings.Join(filters, " and ")) + } + + results := make([]SqlCommand, 0, len(tables)) + for _, table := range tables { + description := fmt.Sprintf("Create View for table %s", table) + command := fmt.Sprintf("create or replace view %s as select * from %s.%s%s", table, pconstants.DuckLakeCatalog, table, filterString) + results = append(results, SqlCommand{Description: description, Command: command}) + } + return results, nil +} diff --git a/internal/display/partition.go b/internal/display/partition.go index 2249cfc4..da38938c 100644 --- a/internal/display/partition.go +++ b/internal/display/partition.go @@ -3,12 +3,9 @@ package display import ( "context" "fmt" - "strings" - "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/filepaths" ) // PartitionResource represents a partition resource and is used for list/show commands @@ -17,6 +14,18 @@ type PartitionResource struct { Description *string `json:"description,omitempty"` Plugin string `json:"plugin"` Local TableResourceFiles `json:"local,omitempty"` + table string + partition string +} + +func NewPartitionResource(p *config.Partition) *PartitionResource { + return &PartitionResource{ + Name: p.UnqualifiedName, + Description: p.Description, + Plugin: p.Plugin.Alias, + table: p.TableName, + partition: p.ShortName, + } } // GetShowData implements the printers.Showable interface @@ -43,10 +52,11 @@ func (r *PartitionResource) GetListData() *printers.RowData { return res } -func ListPartitionResources(ctx context.Context) ([]*PartitionResource, error) { +func ListPartitionResources(ctx context.Context, db *database.DuckDb) ([]*PartitionResource, error) { var res []*PartitionResource - // TODO Add in unconfigured partitions to list output + // TODO Add in unconfigured partitions which exist in database but not configt to list output + // https://github.com/turbot/tailpipe/issues/254 // load all partition names from the data //partitionNames, err := database.ListPartitions(ctx) //if err != nil { @@ -56,14 +66,10 @@ func ListPartitionResources(ctx context.Context) ([]*PartitionResource, error) { partitions := config.GlobalConfig.Partitions for _, p := range partitions { - name := fmt.Sprintf("%s.%s", p.TableName, p.ShortName) - partition := &PartitionResource{ - Name: name, - Description: p.Description, - Plugin: p.Plugin.Alias, - } + partition := NewPartitionResource(p) - err := partition.setFileInformation() + // populate the partition resource with local file information + err := partition.setFileInformation(ctx, db) if err != nil { return nil, fmt.Errorf("error setting file information: %w", err) } @@ -74,18 +80,10 @@ func ListPartitionResources(ctx context.Context) ([]*PartitionResource, error) { return res, nil } -func GetPartitionResource(partitionName string) (*PartitionResource, error) { - p, ok := config.GlobalConfig.Partitions[partitionName] - if !ok { - return nil, fmt.Errorf("no partitions found") - } - partition := &PartitionResource{ - Name: partitionName, - Description: p.Description, - Plugin: p.Plugin.Alias, - } +func GetPartitionResource(ctx context.Context, p *config.Partition, db *database.DuckDb) (*PartitionResource, error) { + partition := NewPartitionResource(p) - err := partition.setFileInformation() + err := partition.setFileInformation(ctx, db) if err != nil { return nil, fmt.Errorf("error setting file information: %w", err) } @@ -93,27 +91,17 @@ func GetPartitionResource(partitionName string) (*PartitionResource, error) { return partition, nil } -func (r *PartitionResource) setFileInformation() error { - dataDir := config.GlobalWorkspaceProfile.GetDataDir() - - nameParts := strings.Split(r.Name, ".") +func (r *PartitionResource) setFileInformation(ctx context.Context, db *database.DuckDb) error { - partitionDir := filepaths.GetParquetPartitionPath(dataDir, nameParts[0], nameParts[1]) - metadata, err := getFileMetadata(partitionDir) + // Get file metadata using shared function + metadata, err := database.GetPartitionFileMetadata(ctx, r.table, r.partition, db) if err != nil { - return err + return fmt.Errorf("unable to obtain file metadata: %w", err) } - r.Local.FileMetadata = metadata - - if metadata.FileCount > 0 { - var rc int64 - rc, err = database.GetRowCount(context.Background(), nameParts[0], &nameParts[1]) - if err != nil { - return fmt.Errorf("unable to obtain row count: %w", err) - } - r.Local.RowCount = rc - } + r.Local.FileSize = metadata.FileSize + r.Local.FileCount = metadata.FileCount + r.Local.RowCount = metadata.RowCount return nil } diff --git a/internal/display/plugin.go b/internal/display/plugin.go index 968928cb..56d49ddc 100644 --- a/internal/display/plugin.go +++ b/internal/display/plugin.go @@ -29,7 +29,7 @@ func (r *PluginListDetails) GetListData() *printers.RowData { func (r *PluginListDetails) setPartitions() { for _, partition := range config.GlobalConfig.Partitions { - if partition.Plugin.Plugin == r.Name { + if partition.Plugin.Plugin == r.Name || isLocalPluginPartition(r, partition.Plugin.Alias) { r.Partitions = append(r.Partitions, strings.TrimPrefix(partition.FullName, "partition.")) } } @@ -37,6 +37,18 @@ func (r *PluginListDetails) setPartitions() { slices.Sort(r.Partitions) } +// handle local plugins: r.Name (from filesystem) can be like "local/plugin-name" +// while partition.Plugin.Plugin is a full image ref like +// "hub.tailpipe.io/plugins/plugin-name/test@latest"; compare alias to last path segment +func isLocalPluginPartition(r *PluginListDetails, partitionAlias string) bool { + return r.Version == "local" && lastSegment(r.Name) == partitionAlias +} + +func lastSegment(s string) string { + p := strings.Split(strings.Trim(s, "/"), "/") + return p[len(p)-1] +} + func ListPlugins(ctx context.Context) ([]*PluginListDetails, error) { var res []*PluginListDetails diff --git a/internal/display/shared.go b/internal/display/shared.go index 1f1631ed..9bcba86c 100644 --- a/internal/display/shared.go +++ b/internal/display/shared.go @@ -1,18 +1,10 @@ package display import ( - "math" - "os" - "path/filepath" - "github.com/dustin/go-humanize" + "math" ) -type FileMetadata struct { - FileSize int64 `json:"file_size"` - FileCount int64 `json:"file_count"` -} - func humanizeBytes(bytes int64) string { if bytes == 0 { return "-" @@ -26,30 +18,3 @@ func humanizeCount(count int64) string { } return humanize.Comma(count) } - -func getFileMetadata(basePath string) (FileMetadata, error) { - var metadata FileMetadata - - // if basePath doesn't exist - nothing collected so short-circuit - if _, err := os.Stat(basePath); os.IsNotExist(err) { - return metadata, nil - } - - // Get File Information - err := filepath.Walk(basePath, func(filePath string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if info.IsDir() { - return nil - } - - metadata.FileCount++ - metadata.FileSize += info.Size() - - return nil - }) - - return metadata, err -} diff --git a/internal/display/source.go b/internal/display/source.go index dd7a7802..5d7eddfc 100644 --- a/internal/display/source.go +++ b/internal/display/source.go @@ -5,26 +5,49 @@ import ( "fmt" "github.com/turbot/pipe-fittings/v2/printers" + "github.com/turbot/tailpipe-plugin-sdk/types" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/plugin" ) type SourceResource struct { - Name string `json:"name"` - Description string `json:"description,omitempty"` - Plugin string `json:"plugin,omitempty"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Plugin string `json:"plugin,omitempty"` + Properties map[string]*types.PropertyMetadata `json:"properties,omitempty"` } // GetShowData implements the printers.Showable interface func (r *SourceResource) GetShowData() *printers.RowData { - res := printers.NewRowData( + var allProperties = []printers.FieldValue{ printers.NewFieldValue("Name", r.Name), printers.NewFieldValue("Plugin", r.Plugin), printers.NewFieldValue("Description", r.Description), - ) + } + if len(r.Properties) > 0 { + allProperties = append(allProperties, printers.NewFieldValue("Properties", r.propertyShowMap())) + } + res := printers.NewRowData(allProperties...) return res } +// GetShowData builds a map of property descriptions to pass to NewFieldValue +func (r *SourceResource) propertyShowMap() map[string]string { + args := map[string]string{} + + for k, v := range r.Properties { + propertyString := v.Type + if v.Required { + propertyString += " (required)" + } + if v.Description != "" { + propertyString += "\n " + v.Description + } + args[k] = propertyString + } + return args +} + // GetListData implements the printers.Listable interface func (r *SourceResource) GetListData() *printers.RowData { res := printers.NewRowData( @@ -65,17 +88,30 @@ func ListSourceResources(ctx context.Context) ([]*SourceResource, error) { } func GetSourceResource(ctx context.Context, sourceName string) (*SourceResource, error) { - // TODO: #refactor simplify by obtaining correct plugin and then extracting it's source - allSources, err := ListSourceResources(ctx) + + // get the plugin which provides the format (note config.GetPluginForFormatByName is smart enough + // to check whether this format is a preset and if so, return the plugin which provides the preset) + pluginName := config.GetPluginForSourceType(sourceName, config.GlobalConfig.PluginVersions) + + // describe plugin to get format info, passing the custom formats list in case this is custom + // (we will have determined this above) + pm := plugin.NewPluginManager() + defer pm.Close() + + desc, err := pm.Describe(ctx, pluginName) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to describe source '%s': %w", sourceName, err) } - for _, source := range allSources { - if source.Name == sourceName { - return source, nil - } + source, ok := desc.Sources[sourceName] + if !ok { + return nil, fmt.Errorf("source '%s' not found", sourceName) } + return &SourceResource{ + Name: source.Name, + Description: source.Description, + Properties: source.Properties, + Plugin: pluginName, + }, nil - return nil, fmt.Errorf("source %s not found", sourceName) } diff --git a/internal/display/table.go b/internal/display/table.go index d2c989fc..28bfbd5b 100644 --- a/internal/display/table.go +++ b/internal/display/table.go @@ -3,14 +3,13 @@ package display import ( "context" "fmt" - "path" "slices" "strings" "github.com/turbot/go-kit/types" "github.com/turbot/pipe-fittings/v2/printers" "github.com/turbot/pipe-fittings/v2/sanitize" - sdkconstants "github.com/turbot/tailpipe-plugin-sdk/constants" + "github.com/turbot/tailpipe-plugin-sdk/helpers" "github.com/turbot/tailpipe-plugin-sdk/schema" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" @@ -29,7 +28,7 @@ type TableResource struct { } // tableResourceFromConfigTable creates a TableResource (display item) from a config.Table (custom table) -func tableResourceFromConfigTable(tableName string, configTable *config.Table) (*TableResource, error) { +func tableResourceFromConfigTable(ctx context.Context, tableName string, configTable *config.Table, db *database.DuckDb) (*TableResource, error) { cols := make([]TableColumnResource, len(configTable.Columns)) for i, c := range configTable.Columns { cols[i] = TableColumnResource{ @@ -42,12 +41,12 @@ func tableResourceFromConfigTable(tableName string, configTable *config.Table) ( table := &TableResource{ Name: tableName, Description: types.SafeString(configTable.Description), - Plugin: constants.CorePluginFullName, + Plugin: constants.CorePluginFullName(), Columns: cols, } table.setPartitions() - err := table.setFileInformation() + err := table.setFileInformation(ctx, db) if err != nil { return nil, fmt.Errorf("failed to set file information for table '%s': %w", tableName, err) } @@ -56,7 +55,7 @@ func tableResourceFromConfigTable(tableName string, configTable *config.Table) ( } // tableResourceFromSchemaTable creates a TableResource (display item) from a schema.TableSchema (defined table) -func tableResourceFromSchemaTable(tableName string, pluginName string, schemaTable *schema.TableSchema) (*TableResource, error) { +func tableResourceFromSchemaTable(ctx context.Context, tableName string, pluginName string, schemaTable *schema.TableSchema, db *database.DuckDb) (*TableResource, error) { cols := make([]TableColumnResource, len(schemaTable.Columns)) for i, c := range schemaTable.Columns { cols[i] = TableColumnResource{ @@ -74,7 +73,7 @@ func tableResourceFromSchemaTable(tableName string, pluginName string, schemaTab } table.setPartitions() - err := table.setFileInformation() + err := table.setFileInformation(ctx, db) if err != nil { return nil, fmt.Errorf("failed to set file information for table '%s': %w", tableName, err) } @@ -91,8 +90,9 @@ type TableColumnResource struct { // TableResourceFiles represents the file information and a row count for a table resource type TableResourceFiles struct { - FileMetadata - RowCount int64 `json:"row_count,omitempty"` + FileSize int64 `json:"file_size"` + FileCount int64 `json:"file_count"` + RowCount int64 `json:"row_count,omitempty"` } // GetShowData implements the printers.Showable interface @@ -123,7 +123,7 @@ func (r *TableResource) GetListData() *printers.RowData { return res } -func ListTableResources(ctx context.Context) ([]*TableResource, error) { +func ListTableResources(ctx context.Context, db *database.DuckDb) ([]*TableResource, error) { var res []*TableResource tables := make(map[string]*TableResource) @@ -136,25 +136,25 @@ func ListTableResources(ctx context.Context) ([]*TableResource, error) { return nil, fmt.Errorf("unable to obtain plugin list: %w", err) } - for _, p := range plugins { - desc, err := pluginManager.Describe(ctx, p.Name) + for _, partition := range plugins { + desc, err := pluginManager.Describe(ctx, partition.Name) if err != nil { return nil, fmt.Errorf("unable to obtain plugin details: %w", err) } - for t, s := range desc.Schemas { - table, err := tableResourceFromSchemaTable(t, p.Name, s) + for tableName, schema := range desc.Schemas { + table, err := tableResourceFromSchemaTable(ctx, tableName, partition.Name, schema, db) if err != nil { return nil, err } - tables[t] = table + tables[tableName] = table } } // custom tables - these take precedence over plugin defined tables, so overwrite any duplicates in map for tableName, tableDef := range config.GlobalConfig.CustomTables { - table, err := tableResourceFromConfigTable(tableName, tableDef) + table, err := tableResourceFromConfigTable(ctx, tableName, tableDef, db) if err != nil { return nil, err } @@ -170,10 +170,10 @@ func ListTableResources(ctx context.Context) ([]*TableResource, error) { return res, nil } -func GetTableResource(ctx context.Context, tableName string) (*TableResource, error) { +func GetTableResource(ctx context.Context, tableName string, db *database.DuckDb) (*TableResource, error) { // custom table takes precedence over plugin defined table, check there first if customTable, ok := config.GlobalConfig.CustomTables[tableName]; ok { - table, err := tableResourceFromConfigTable(tableName, customTable) + table, err := tableResourceFromConfigTable(ctx, tableName, customTable, db) return table, err } @@ -185,7 +185,7 @@ func GetTableResource(ctx context.Context, tableName string) (*TableResource, er // if this is a custom table, we need to use the core plugin // NOTE: we cannot do this inside GetPluginForTable as that funciton may be called before the config is fully populated if _, isCustom := config.GlobalConfig.CustomTables[tableName]; isCustom { - pluginName = constants.CorePluginName + pluginName = constants.CorePluginInstallStream() } desc, err := pluginManager.Describe(ctx, pluginName) @@ -194,7 +194,7 @@ func GetTableResource(ctx context.Context, tableName string) (*TableResource, er } if tableSchema, ok := desc.Schemas[tableName]; ok { - return tableResourceFromSchemaTable(tableName, pluginName, tableSchema) + return tableResourceFromSchemaTable(ctx, tableName, pluginName, tableSchema, db) } else { return nil, fmt.Errorf("table %s not found", tableName) } @@ -210,22 +210,16 @@ func (r *TableResource) setPartitions() { slices.Sort(r.Partitions) } -func (r *TableResource) setFileInformation() error { - metadata, err := getFileMetadata(path.Join(config.GlobalWorkspaceProfile.GetDataDir(), fmt.Sprintf("%s=%s", sdkconstants.TpTable, r.Name))) +func (r *TableResource) setFileInformation(ctx context.Context, db *database.DuckDb) error { + // Get file metadata using shared function + metadata, err := database.GetTableFileMetadata(ctx, r.Name, db) if err != nil { return fmt.Errorf("unable to obtain file metadata: %w", err) } - r.Local.FileMetadata = metadata - - if metadata.FileCount > 0 { - var rc int64 - rc, err = database.GetRowCount(context.Background(), r.Name, nil) - if err != nil { - return fmt.Errorf("unable to obtain row count: %w", err) - } - r.Local.RowCount = rc - } + r.Local.FileSize = metadata.FileSize + r.Local.FileCount = metadata.FileCount + r.Local.RowCount = metadata.RowCount return nil } @@ -235,24 +229,21 @@ func (r *TableResource) getColumnsRenderFunc() printers.RenderFunc { var lines []string lines = append(lines, "") // blank line before column details - cols := r.Columns - // TODO: #graza we utilize similar behaviour in the view creation but only on string, can we combine these into a single func? - tpPrefix := "tp_" - slices.SortFunc(cols, func(a, b TableColumnResource) int { - isPrefixedA, isPrefixedB := strings.HasPrefix(a.ColumnName, tpPrefix), strings.HasPrefix(b.ColumnName, tpPrefix) - switch { - case isPrefixedA && !isPrefixedB: - return 1 // a > b - case !isPrefixedA && isPrefixedB: - return -1 // a < b - default: - return strings.Compare(a.ColumnName, b.ColumnName) // standard alphabetical sort - } - }) + // Extract column names and build map in a single loop + columnNames := make([]string, len(r.Columns)) + columnMap := make(map[string]TableColumnResource) + for i, col := range r.Columns { + columnNames[i] = col.ColumnName + columnMap[col.ColumnName] = col + } + // sort column names alphabetically, with tp fields at the end + sortedColumnNames := helpers.SortColumnsAlphabetically(columnNames) - for _, c := range r.Columns { + // Build lines in sorted order + for _, colName := range sortedColumnNames { + col := columnMap[colName] // type is forced to lowercase, this should be the case for our tables/plugins but this provides consistency for custom tables, etc - line := fmt.Sprintf(" %s: %s", c.ColumnName, strings.ToLower(c.Type)) + line := fmt.Sprintf(" %s: %s", col.ColumnName, strings.ToLower(col.Type)) lines = append(lines, line) } diff --git a/internal/error_helpers/error_helpers.go b/internal/error_helpers/error_helpers.go new file mode 100644 index 00000000..8c55d866 --- /dev/null +++ b/internal/error_helpers/error_helpers.go @@ -0,0 +1,124 @@ +// Copied from pipe-fittings/error_helpers.go. We handle cancellation differently: +// cancellations are a user choice, so we don't throw an error (normalized to "execution cancelled"). +// +//nolint:forbidigo // TODO: review fmt usage +package error_helpers + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/fatih/color" + "github.com/shiena/ansicolor" + "github.com/spf13/viper" + "github.com/turbot/pipe-fittings/v2/constants" + "github.com/turbot/pipe-fittings/v2/statushooks" +) + +func init() { + color.Output = ansicolor.NewAnsiColorWriter(os.Stderr) +} + +func FailOnError(err error) { + if err != nil { + panic(err) + } +} + +func FailOnErrorWithMessage(err error, message string) { + if err != nil { + panic(fmt.Sprintf("%s: %s", message, err.Error())) + } +} + +func ShowError(ctx context.Context, err error) { + if err == nil { + return + } + statushooks.Done(ctx) + opStream := GetWarningOutputStream() + fmt.Fprintf(opStream, "%s: %v\n", constants.ColoredErr, TransformErrorToTailpipe(err)) +} + +// ShowErrorWithMessage displays the given error nicely with the given message +func ShowErrorWithMessage(ctx context.Context, err error, message string) { + if err == nil { + return + } + statushooks.Done(ctx) + opStream := GetWarningOutputStream() + fmt.Fprintf(opStream, "%s: %s - %v\n", constants.ColoredErr, message, TransformErrorToTailpipe(err)) +} + +// TransformErrorToTailpipe removes the pq: and rpc error prefixes along +// with all the unnecessary information that comes from the +// drivers and libraries +func TransformErrorToTailpipe(err error) error { + if err == nil { + return nil + } + + var errString string + if strings.Contains(err.Error(), "flowpipe service is unreachable") { + errString = strings.Split(err.Error(), ": ")[1] + } else { + errString = strings.TrimSpace(err.Error()) + } + + // an error that originated from our database/sql driver (always prefixed with "ERROR:") + if strings.HasPrefix(errString, "ERROR:") { + errString = strings.TrimSpace(strings.TrimPrefix(errString, "ERROR:")) + } + // if this is an RPC Error while talking with the plugin + if strings.HasPrefix(errString, "rpc error") { + // trim out "rpc error: code = Unknown desc =" + errString = strings.TrimPrefix(errString, "rpc error: code = Unknown desc =") + } + return errors.New(strings.TrimSpace(errString)) +} + +func IsCancelledError(err error) bool { + return errors.Is(err, context.Canceled) || strings.Contains(err.Error(), "canceling statement due to user request") +} + +func ShowWarning(warning string) { + if len(warning) == 0 { + return + } + opStream := GetWarningOutputStream() + fmt.Fprintf(opStream, "%s: %v\n", constants.ColoredWarn, warning) +} + +// ShowInfo prints a non-critical info message to the appropriate output stream. +// Behaves like ShowWarning but with a calmer label (Note) to avoid alarming users +// for successful outcomes or informational messages. +func ShowInfo(info string) { + if len(info) == 0 { + return + } + opStream := GetWarningOutputStream() + fmt.Fprintf(opStream, "%s: %v\n", color.YellowString("Note"), info) +} + +func PrefixError(err error, prefix string) error { + return fmt.Errorf("%s: %s\n", prefix, TransformErrorToTailpipe(err).Error()) +} + +// IsMachineReadableOutput checks if the current output format is machine readable (CSV or JSON) +func IsMachineReadableOutput() bool { + outputFormat := viper.GetString(constants.ArgOutput) + return outputFormat == constants.OutputFormatCSV || outputFormat == constants.OutputFormatJSON || outputFormat == constants.OutputFormatLine +} + +func GetWarningOutputStream() io.Writer { + if IsMachineReadableOutput() { + // For machine-readable formats, output warnings and errors to stderr + return os.Stderr + } + // For all other formats, use stdout + return os.Stdout +} diff --git a/internal/filepaths/collection_temp_dir.go b/internal/filepaths/collection_temp_dir.go index 1e33b0ee..b635f0ff 100644 --- a/internal/filepaths/collection_temp_dir.go +++ b/internal/filepaths/collection_temp_dir.go @@ -1,56 +1,13 @@ package filepaths import ( - "fmt" - "github.com/turbot/pipe-fittings/v2/utils" + + "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/tailpipe/internal/config" - "log/slog" - "os" - "path/filepath" - "strconv" ) func EnsureCollectionTempDir() string { collectionDir := config.GlobalWorkspaceProfile.GetCollectionDir() - - // add a PID directory to the collection directory - collectionTempDir := filepath.Join(collectionDir, fmt.Sprintf("%d", os.Getpid())) - - // create the directory if it doesn't exist - if _, err := os.Stat(collectionTempDir); os.IsNotExist(err) { - err := os.MkdirAll(collectionTempDir, 0755) - if err != nil { - slog.Error("failed to create collection temp dir", "error", err) - } - } - return collectionTempDir -} - -func CleanupCollectionTempDirs() { - // get the collection directory for this workspace - collectionDir := config.GlobalWorkspaceProfile.GetCollectionDir() - - files, err := os.ReadDir(collectionDir) - if err != nil { - slog.Warn("failed to list files in collection dir", "error", err) - return - } - for _, file := range files { - // if the file is a directory and is not our collection temp dir, remove it - if file.IsDir() { - // the folder name is the PID - check whether that pid exists - // if it doesn't, remove the folder - // Attempt to find the process - // try to parse the directory name as a pid - pid, err := strconv.ParseInt(file.Name(), 10, 32) - if err == nil { - if utils.PidExists(int(pid)) { - slog.Info(fmt.Sprintf("Cleaning existing collection temp dirs - skipping directory '%s' as process with PID %d exists", file.Name(), pid)) - continue - } - } - slog.Debug("Removing directory", "dir", file.Name()) - _ = os.RemoveAll(filepath.Join(collectionDir, file.Name())) - } - } + pidTempDir := filepaths.EnsurePidTempDir(collectionDir) + return pidTempDir } diff --git a/internal/filepaths/database.go b/internal/filepaths/database.go deleted file mode 100644 index c11b9cbf..00000000 --- a/internal/filepaths/database.go +++ /dev/null @@ -1,13 +0,0 @@ -package filepaths - -import ( - "path/filepath" - - "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/constants" -) - -func TailpipeDbFilePath() string { - dataDir := config.GlobalWorkspaceProfile.GetDataDir() - return filepath.Join(dataDir, constants.TailpipeDbName) -} diff --git a/internal/filepaths/parquet.go b/internal/filepaths/parquet.go deleted file mode 100644 index 0e4cd22a..00000000 --- a/internal/filepaths/parquet.go +++ /dev/null @@ -1,37 +0,0 @@ -package filepaths - -import ( - "fmt" - - "path/filepath" - - pfilepaths "github.com/turbot/pipe-fittings/v2/filepaths" -) - -const TempParquetExtension = ".parquet.tmp" - -func GetParquetFileGlobForTable(dataDir, tableName, fileRoot string) string { - return filepath.Join(dataDir, fmt.Sprintf("tp_table=%s/*/*/*/%s*.parquet", tableName, fileRoot)) -} - -func GetParquetFileGlobForPartition(dataDir, tableName, partitionName, fileRoot string) string { - return filepath.Join(dataDir, fmt.Sprintf("tp_table=%s/tp_partition=%s/*/*/%s*.parquet", tableName, partitionName, fileRoot)) -} - -func GetTempParquetFileGlobForPartition(dataDir, tableName, partitionName, fileRoot string) string { - return filepath.Join(dataDir, fmt.Sprintf("tp_table=%s/tp_partition=%s/*/*/%s*%s", tableName, partitionName, fileRoot, TempParquetExtension)) -} - -// GetTempAndInvalidParquetFileGlobForPartition returns a glob pattern for invalid and temporary parquet files for a partition -func GetTempAndInvalidParquetFileGlobForPartition(dataDir, tableName, partitionName string) string { - base := filepath.Join(dataDir, fmt.Sprintf("tp_table=%s/tp_partition=%s", tableName, partitionName)) - return filepath.Join(base, "*.parquet.*") -} - -func GetParquetPartitionPath(dataDir, tableName, partitionName string) string { - return filepath.Join(dataDir, fmt.Sprintf("tp_table=%s/tp_partition=%s", tableName, partitionName)) -} - -func InvalidParquetFilePath() string { - return filepath.Join(pfilepaths.EnsureInternalDir(), "invalid_parquet.json") -} diff --git a/internal/filepaths/partition_fields.go b/internal/filepaths/partition_fields.go deleted file mode 100644 index c9e9663a..00000000 --- a/internal/filepaths/partition_fields.go +++ /dev/null @@ -1,72 +0,0 @@ -package filepaths - -import ( - "fmt" - "strconv" - "strings" - "time" -) - -// PartitionFields represents the components of a parquet file path -type PartitionFields struct { - Table string - Partition string - Date time.Time - Index int -} - -// ExtractPartitionFields parses a parquet file path and returns its components. -// Expected path format: -// -// /path/to/dir/tp_table=/tp_partition=/tp_date=/tp_index=/file.parquet -// -// Rules: -// - Fields can appear in any order -// - It is an error for the same field to appear with different values -// - Date must be in YYYY-MM-DD format -// - Missing fields are allowed (will have zero values) -func ExtractPartitionFields(parquetFilePath string) (PartitionFields, error) { - fields := PartitionFields{} - - parts := strings.Split(parquetFilePath, "/") - for _, part := range parts { - switch { - case strings.HasPrefix(part, "tp_table="): - value := strings.TrimPrefix(part, "tp_table=") - if fields.Table != "" && fields.Table != value { - return PartitionFields{}, fmt.Errorf("conflicting table values: %s and %s", fields.Table, value) - } - fields.Table = value - case strings.HasPrefix(part, "tp_partition="): - value := strings.TrimPrefix(part, "tp_partition=") - if fields.Partition != "" && fields.Partition != value { - return PartitionFields{}, fmt.Errorf("conflicting partition values: %s and %s", fields.Partition, value) - } - fields.Partition = value - case strings.HasPrefix(part, "tp_date="): - value := strings.TrimPrefix(part, "tp_date=") - date, err := time.Parse("2006-01-02", value) - if err == nil { - if !fields.Date.IsZero() && !fields.Date.Equal(date) { - return PartitionFields{}, fmt.Errorf("conflicting date values: %s and %s", fields.Date.Format("2006-01-02"), value) - } - fields.Date = date - } - case strings.HasPrefix(part, "tp_index="): - value := strings.TrimPrefix(part, "tp_index=") - if fields.Index != 0 { - if index, err := strconv.Atoi(value); err == nil { - if fields.Index != index { - return PartitionFields{}, fmt.Errorf("conflicting index values: %d and %s", fields.Index, value) - } - } - } else { - if index, err := strconv.Atoi(value); err == nil { - fields.Index = index - } - } - } - } - - return fields, nil -} diff --git a/internal/filepaths/partition_fields_test.go b/internal/filepaths/partition_fields_test.go deleted file mode 100644 index a95118de..00000000 --- a/internal/filepaths/partition_fields_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package filepaths - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestExtractPartitionFields(t *testing.T) { - tests := []struct { - name string - path string - expected PartitionFields - expectError bool - }{ - { - name: "complete path", - path: "/some/path/tp_table=aws_account/tp_partition=123456789/tp_date=2024-03-15/tp_index=1/file.parquet", - expected: PartitionFields{ - Table: "aws_account", - Partition: "123456789", - Date: time.Date(2024, 3, 15, 0, 0, 0, 0, time.UTC), - Index: 1, - }, - expectError: false, - }, - { - name: "missing index", - path: "/path/tp_table=aws_account/tp_partition=123456789/tp_date=2024-03-15/file.parquet", - expected: PartitionFields{ - Table: "aws_account", - Partition: "123456789", - Date: time.Date(2024, 3, 15, 0, 0, 0, 0, time.UTC), - Index: 0, - }, - expectError: false, - }, - { - name: "invalid date", - path: "/path/tp_table=aws_account/tp_partition=123456789/tp_date=invalid/tp_index=1/file.parquet", - expected: PartitionFields{ - Table: "aws_account", - Partition: "123456789", - Date: time.Time{}, - Index: 1, - }, - expectError: false, - }, - { - name: "invalid index", - path: "/path/tp_table=aws_account/tp_partition=123456789/tp_date=2024-03-15/tp_index=invalid/file.parquet", - expected: PartitionFields{ - Table: "aws_account", - Partition: "123456789", - Date: time.Date(2024, 3, 15, 0, 0, 0, 0, time.UTC), - Index: 0, - }, - expectError: false, - }, - { - name: "empty path", - path: "", - expected: PartitionFields{}, - expectError: false, - }, - { - name: "duplicate table field with different values", - path: "/path/tp_table=aws_account/tp_table=aws_iam/tp_partition=123456789/tp_date=2024-03-15/tp_index=1/file.parquet", - expected: PartitionFields{}, - expectError: true, - }, - { - name: "duplicate partition field with different values", - path: "/path/tp_table=aws_account/tp_partition=123456789/tp_partition=987654321/tp_date=2024-03-15/tp_index=1/file.parquet", - expected: PartitionFields{}, - expectError: true, - }, - { - name: "duplicate date field with different values", - path: "/path/tp_table=aws_account/tp_partition=123456789/tp_date=2024-03-15/tp_date=2024-03-16/tp_index=1/file.parquet", - expected: PartitionFields{}, - expectError: true, - }, - { - name: "duplicate index field with different values", - path: "/path/tp_table=aws_account/tp_partition=123456789/tp_date=2024-03-15/tp_index=1/tp_index=2/file.parquet", - expected: PartitionFields{}, - expectError: true, - }, - { - name: "duplicate fields with same values should not error", - path: "/path/tp_table=aws_account/tp_table=aws_account/tp_partition=123456789/tp_partition=123456789/tp_date=2024-03-15/tp_date=2024-03-15/tp_index=1/tp_index=1/file.parquet", - expected: PartitionFields{ - Table: "aws_account", - Partition: "123456789", - Date: time.Date(2024, 3, 15, 0, 0, 0, 0, time.UTC), - Index: 1, - }, - expectError: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result, err := ExtractPartitionFields(tt.path) - if tt.expectError { - assert.Error(t, err) - assert.Empty(t, result) - return - } - assert.NoError(t, err) - assert.Equal(t, tt.expected, result) - }) - } -} diff --git a/internal/filepaths/prune.go b/internal/filepaths/prune.go index e4c73f13..247df1a3 100644 --- a/internal/filepaths/prune.go +++ b/internal/filepaths/prune.go @@ -1,9 +1,9 @@ package filepaths import ( - "io" "os" "path/filepath" + pfilepaths "github.com/turbot/pipe-fittings/v2/filepaths" ) // PruneTree recursively deletes empty directories in the given folder. @@ -12,7 +12,7 @@ func PruneTree(folder string) error { if _, err := os.Stat(folder); os.IsNotExist(err) { return nil } - isEmpty, err := isDirEmpty(folder) + isEmpty, err := pfilepaths.IsDirEmpty(folder) if err != nil { return err } @@ -36,7 +36,7 @@ func PruneTree(folder string) error { } // Check again if the folder is empty after pruning subdirectories - isEmpty, err = isDirEmpty(folder) + isEmpty, err = pfilepaths.IsDirEmpty(folder) if err != nil { return err } @@ -47,18 +47,3 @@ func PruneTree(folder string) error { return nil } - -// isDirEmpty checks if a directory is empty. -func isDirEmpty(dir string) (bool, error) { - f, err := os.Open(dir) - if err != nil { - return false, err - } - defer f.Close() - - _, err = f.Readdir(1) - if err == io.EOF { - return true, nil - } - return false, err -} diff --git a/internal/helpers/errors.go b/internal/helpers/errors.go new file mode 100644 index 00000000..20a4e737 --- /dev/null +++ b/internal/helpers/errors.go @@ -0,0 +1,16 @@ +package helpers + +import ( + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// IsNotGRPCImplementedError checks if the provided error is a gRPC 'Unimplemented' status error. +func IsNotGRPCImplementedError(err error) bool { + status, ok := status.FromError(err) + if !ok { + return false + } + + return status.Code() == codes.Unimplemented +} diff --git a/internal/interactive/interactive_client.go b/internal/interactive/interactive_client.go index b358f47f..6e05cafc 100644 --- a/internal/interactive/interactive_client.go +++ b/internal/interactive/interactive_client.go @@ -17,10 +17,10 @@ import ( "github.com/spf13/viper" "github.com/turbot/go-kit/helpers" pconstants "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/statushooks" "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/tailpipe/internal/database" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" "github.com/turbot/tailpipe/internal/metaquery" "github.com/turbot/tailpipe/internal/query" ) @@ -48,7 +48,7 @@ type InteractiveClient struct { executionLock sync.Mutex // the schema metadata - this is loaded asynchronously during init //schemaMetadata *db_common.SchemaMetadata - tableViews []string + tables []string highlighter *Highlighter // hidePrompt is used to render a blank as the prompt prefix hidePrompt bool @@ -79,12 +79,12 @@ func newInteractiveClient(ctx context.Context, db *database.DuckDb) (*Interactiv db: db, } - // initialise the table views for autocomplete - tv, err := database.GetTableViews(ctx) + // initialise the table list for autocomplete + tv, err := database.GetTables(ctx, db) if err != nil { return nil, err } - c.tableViews = tv + c.tables = tv // initialise autocomplete suggestions err = c.initialiseSuggestions(ctx) @@ -346,7 +346,7 @@ func (c *InteractiveClient) executor(ctx context.Context, line string) { func (c *InteractiveClient) executeQuery(ctx context.Context, queryCtx context.Context, resolvedQuery *ResolvedQuery) { _, err := query.ExecuteQuery(queryCtx, resolvedQuery.ExecuteSQL, c.db) if err != nil { - error_helpers.ShowError(ctx, error_helpers.HandleCancelError(err)) + error_helpers.ShowError(ctx, err) } } @@ -434,6 +434,7 @@ func (c *InteractiveClient) executeMetaquery(ctx context.Context, query string) Query: query, Prompt: c.interactivePrompt, ClosePrompt: func() { c.afterClose = AfterPromptCloseExit }, + Db: c.db, }) } @@ -477,15 +478,18 @@ func (c *InteractiveClient) queryCompleter(d prompt.Document) []prompt.Suggest { case isFirstWord(text): suggestions := c.getFirstWordSuggestions(text) s = append(s, suggestions...) + case isDuckDbMetaQuery(text): + tableSuggestions := c.getTableSuggestions() + s = append(s, tableSuggestions...) case metaquery.IsMetaQuery(text): suggestions := metaquery.Complete(&metaquery.CompleterInput{ Query: text, - ViewSuggestions: c.getTableSuggestions(lastWord(text)), + ViewSuggestions: c.getTableSuggestions(), }) s = append(s, suggestions...) default: if queryInfo := getQueryInfo(text); queryInfo.EditingTable { - tableSuggestions := c.getTableSuggestions(lastWord(text)) + tableSuggestions := c.getTableSuggestions() s = append(s, tableSuggestions...) } } @@ -497,31 +501,30 @@ func (c *InteractiveClient) getFirstWordSuggestions(word string) []prompt.Sugges var s []prompt.Suggest // add all we know that can be the first words - // "select", "with" - s = append(s, prompt.Suggest{Text: "select", Output: "select"}, prompt.Suggest{Text: "with", Output: "with"}) + // "select", "with", "describe", "show", "summarize" + s = append(s, + prompt.Suggest{Text: "select", Output: "select"}, + prompt.Suggest{Text: "with", Output: "with"}, + prompt.Suggest{Text: "describe", Output: "describe"}, + prompt.Suggest{Text: "show", Output: "show"}, + prompt.Suggest{Text: "summarize", Output: "summarize"}, + prompt.Suggest{Text: "explain", Output: "explain"}, + ) // metaqueries s = append(s, metaquery.PromptSuggestions()...) return s } -func (c *InteractiveClient) getTableSuggestions(word string) []prompt.Suggest { +func (c *InteractiveClient) getTableSuggestions() []prompt.Suggest { var s []prompt.Suggest - for _, tv := range c.tableViews { - s = append(s, prompt.Suggest{Text: tv, Output: tv}) + for _, tableName := range c.tables { + s = append(s, prompt.Suggest{Text: tableName, Output: tableName}) } return s } -// -//func (c *InteractiveClient) newSuggestion(itemType string, description string, name string) prompt.Suggest { -// if description != "" { -// itemType += fmt.Sprintf(": %s", description) -// } -// return prompt.Suggest{Text: name, Output: name, Description: itemType} -//} - func (c *InteractiveClient) startCancelHandler() chan bool { sigIntChannel := make(chan os.Signal, 1) quitChannel := make(chan bool, 1) diff --git a/internal/interactive/interactive_client_autocomplete.go b/internal/interactive/interactive_client_autocomplete.go index bab662cb..5f2b86c1 100644 --- a/internal/interactive/interactive_client_autocomplete.go +++ b/internal/interactive/interactive_client_autocomplete.go @@ -2,12 +2,9 @@ package interactive import ( "context" - "log" ) func (c *InteractiveClient) initialiseSuggestions(ctx context.Context) error { - log.Printf("[TRACE] initialiseSuggestions") - // reset suggestions c.suggestions = newAutocompleteSuggestions() c.suggestions.sort() diff --git a/internal/interactive/interactive_helpers.go b/internal/interactive/interactive_helpers.go index 99bb80ea..34932e7a 100644 --- a/internal/interactive/interactive_helpers.go +++ b/internal/interactive/interactive_helpers.go @@ -71,9 +71,19 @@ func isFirstWord(text string) bool { return strings.LastIndex(text, " ") == -1 } -// split the string by spaces and return the last segment -func lastWord(text string) string { - return text[strings.LastIndex(text, " "):] +// isDuckDbMetaQuery returns true if the input string equals 'describe', 'show', or 'summarize' +func isDuckDbMetaQuery(s string) bool { + ts := strings.ToLower(strings.TrimSpace(s)) + switch { + case ts == "describe": + return true + case ts == "show": + return true + case ts == "summarize": + return true + default: + return false + } } // diff --git a/internal/interactive/run.go b/internal/interactive/run.go index 2bfa5faa..4c17435e 100644 --- a/internal/interactive/run.go +++ b/internal/interactive/run.go @@ -3,8 +3,8 @@ package interactive import ( "context" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/tailpipe/internal/database" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" ) // RunInteractivePrompt starts the interactive query prompt diff --git a/internal/metaquery/handler_input.go b/internal/metaquery/handler_input.go index 53fea612..11af2f65 100644 --- a/internal/metaquery/handler_input.go +++ b/internal/metaquery/handler_input.go @@ -14,20 +14,21 @@ type HandlerInput struct { ClosePrompt func() Query string - views *[]string + tables *[]string + Db *database.DuckDb } func (h *HandlerInput) args() []string { return getArguments(h.Query) } -func (h *HandlerInput) GetViews() ([]string, error) { - if h.views == nil { - views, err := database.GetTableViews(context.Background()) +func (h *HandlerInput) GetTables(ctx context.Context) ([]string, error) { + if h.tables == nil { + tables, err := database.GetTables(ctx, h.Db) if err != nil { return nil, err } - h.views = &views + h.tables = &tables } - return *h.views, nil + return *h.tables, nil } diff --git a/internal/metaquery/handler_inspect.go b/internal/metaquery/handler_inspect.go index a5d6b7b1..fe4c09a0 100644 --- a/internal/metaquery/handler_inspect.go +++ b/internal/metaquery/handler_inspect.go @@ -3,40 +3,42 @@ package metaquery import ( "context" "fmt" - "github.com/turbot/tailpipe/internal/plugin" "slices" - "sort" "strings" + "github.com/turbot/tailpipe-plugin-sdk/helpers" "github.com/turbot/tailpipe/internal/config" + "github.com/turbot/tailpipe/internal/constants" "github.com/turbot/tailpipe/internal/database" + "github.com/turbot/tailpipe/internal/plugin" ) // inspect func inspect(ctx context.Context, input *HandlerInput) error { - views, err := input.GetViews() + tables, err := input.GetTables(ctx) if err != nil { return fmt.Errorf("failed to get tables: %w", err) } if len(input.args()) == 0 { - return listViews(ctx, input, views) + return listTables(ctx, input, tables) } - viewName := input.args()[0] - if slices.Contains(views, viewName) { - return listViewSchema(ctx, input, viewName) + tableName := input.args()[0] + if slices.Contains(tables, tableName) { + return getTableSchema(ctx, input, tableName) } - return fmt.Errorf("could not find a view named '%s'", viewName) + return fmt.Errorf("could not find a view named '%s'", tableName) } -func listViews(ctx context.Context, input *HandlerInput, views []string) error { +func listTables(ctx context.Context, input *HandlerInput, views []string) error { var rows [][]string rows = append(rows, []string{"Table", "Plugin"}) // Header for _, view := range views { // TODO look at using config.GetPluginForTable(ctx, view) instead of this - or perhaps add function + // https://github.com/turbot/tailpipe/issues/500 // GetPluginAndVersionForTable? // getPluginForTable looks at plugin binaries (slower but mre reliable) p, _ := getPluginForTable(ctx, view) @@ -47,10 +49,10 @@ func listViews(ctx context.Context, input *HandlerInput, views []string) error { return nil } -func listViewSchema(ctx context.Context, input *HandlerInput, viewName string) error { - schema, err := database.GetTableViewSchema(ctx, viewName) +func getTableSchema(ctx context.Context, input *HandlerInput, tableName string) error { + schema, err := database.GetTableSchema(ctx, tableName, input.Db) if err != nil { - return fmt.Errorf("failed to get view schema: %w", err) + return fmt.Errorf("failed to get table schema: %w", err) } var rows [][]string @@ -60,7 +62,9 @@ func listViewSchema(ctx context.Context, input *HandlerInput, viewName string) e for column := range schema { cols = append(cols, column) } - sort.Strings(cols) + + // Sort column names alphabetically but with tp_ fields on the end + cols = helpers.SortColumnsAlphabetically(cols) for _, col := range cols { rows = append(rows, []string{col, strings.ToLower(schema[col])}) @@ -73,6 +77,13 @@ func listViewSchema(ctx context.Context, input *HandlerInput, viewName string) e // getPluginForTable returns the plugin name and version for a given table name. // note - this looks at the installed plugins and their version file entry, not only the version file func getPluginForTable(ctx context.Context, tableName string) (string, error) { + // First check if this is a custom table + if _, isCustom := config.GlobalConfig.CustomTables[tableName]; isCustom { + // Custom tables use the core plugin + corePluginName := constants.CorePluginInstallStream() + return corePluginName, nil + } + prefix := strings.Split(tableName, "_")[0] ps, err := plugin.GetInstalledPlugins(ctx, config.GlobalConfig.PluginVersions) diff --git a/internal/migration/error.go b/internal/migration/error.go new file mode 100644 index 00000000..eac57f06 --- /dev/null +++ b/internal/migration/error.go @@ -0,0 +1,33 @@ +package migration + +import ( + "fmt" +) + +// MigrationError is an aggregate error that wraps multiple child errors +// encountered during migration. +type MigrationError struct { + errors []error +} + +func NewMigrationError() *MigrationError { + return &MigrationError{errors: make([]error, 0)} +} + +func (m *MigrationError) Append(err error) { + if err == nil { + return + } + m.errors = append(m.errors, err) +} + +func (m *MigrationError) Len() int { return len(m.errors) } + +// Error provides a compact summary string +func (m *MigrationError) Error() string { + return fmt.Sprintf("%d error(s) occurred during migration", len(m.errors)) +} + +// Unwrap returns the list of child errors so that errors.Is/As can walk them +// (supported since Go 1.20 with Unwrap() []error) +func (m *MigrationError) Unwrap() []error { return m.errors } diff --git a/internal/migration/errors.go b/internal/migration/errors.go new file mode 100644 index 00000000..231ba6f0 --- /dev/null +++ b/internal/migration/errors.go @@ -0,0 +1,27 @@ +package migration + +import "fmt" + +// UnsupportedError represents an error when migration is not supported +// due to specific command line arguments or configuration +type UnsupportedError struct { + Reason string +} + +func (e *UnsupportedError) Error() string { + msgFormat := "data must be migrated to Ducklake format - migration is not supported with '%s'.\n\nRun 'tailpipe query' to migrate your data to DuckLake format" + return fmt.Sprintf(msgFormat, e.Reason) +} + +func (e *UnsupportedError) Is(target error) bool { + _, ok := target.(*UnsupportedError) + return ok +} + +func (e *UnsupportedError) As(target interface{}) bool { + if t, ok := target.(**UnsupportedError); ok { + *t = e + return true + } + return false +} diff --git a/internal/migration/migration.go b/internal/migration/migration.go new file mode 100644 index 00000000..08893322 --- /dev/null +++ b/internal/migration/migration.go @@ -0,0 +1,659 @@ +package migration + +import ( + "bufio" + "context" + "database/sql" + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + "strings" + "time" + + "github.com/briandowns/spinner" + "github.com/spf13/viper" + "github.com/turbot/pipe-fittings/v2/constants" + perr "github.com/turbot/pipe-fittings/v2/error_helpers" + "github.com/turbot/pipe-fittings/v2/utils" + "github.com/turbot/tailpipe-plugin-sdk/schema" + "github.com/turbot/tailpipe/internal/config" + "github.com/turbot/tailpipe/internal/database" + "github.com/turbot/tailpipe/internal/error_helpers" + "github.com/turbot/tailpipe/internal/filepaths" +) + +// StatusType represents different types of migration status messages +type StatusType int + +const ( + InitialisationFailed StatusType = iota + MigrationFailed + CleanupAfterSuccess + PartialSuccess + Success +) + +// MigrateDataToDucklake performs migration of views from tailpipe.db and associated parquet files +// into the new DuckLake metadata catalog +func MigrateDataToDucklake(ctx context.Context) (err error) { + slog.Info("Starting data migration to DuckLake format") + // define a status message var - this will be set when we encounter any issues - or when we are successful + // this will be printed at the end of the function + var statusMsg string + var partialMigrated bool + + // if there is a status message, print it out at the end + defer func() { + if statusMsg != "" { + if err != nil || partialMigrated { + // if there is an error or a partial migration, show as warning + error_helpers.ShowWarning(statusMsg) + } else { + // show as info if there is no error, or if it is not a partial migration + error_helpers.ShowInfo(statusMsg) + } + } + }() + + // Determine source and migration directories + dataDefaultDir := config.GlobalWorkspaceProfile.GetDataDir() + migratingDefaultDir := config.GlobalWorkspaceProfile.GetMigratingDir() + + var matchedTableDirs, unmatchedTableDirs []string + + // if the ~/.tailpipe/data directory has a .db file, it means that this is the first time we are migrating + // if the ~/.tailpipe/migration/migrating directory has a .db file, it means that this is a resume migration + initialMigration := hasTailpipeDb(dataDefaultDir) + continueMigration := hasTailpipeDb(migratingDefaultDir) + + // validate: both should not be true - return that last migration left things in a bad state + if initialMigration && continueMigration { + return fmt.Errorf("invalid migration state: found tailpipe.db in both data and migrating directories. This should not happen. Please contact Turbot support for assistance") + } + + // STEP 1: Check if migration is needed + // We need to migrate if it is the first time we are migrating or if we are resuming a migration + if !initialMigration && !continueMigration { + slog.Info("No migration needed - no tailpipe.db found in data or migrating directory") + return nil + } + + // if the output for this command is a machine readable format (csv/json) or progress is false, + // it is possible/likely that tailpipe is being used in a non interactive way - in this case, + // we should not prompt the user, instead return an error + if err := checkMigrationSupported(); err != nil { + return err + } + + // Prompt the user to confirm migration + shouldContinue, err := promptUserForMigration(ctx, dataDefaultDir) + if err != nil { + return fmt.Errorf("failed to get user confirmation: %w", err) + } + if !shouldContinue { + return context.Canceled + } + + logPath := filepath.Join(config.GlobalWorkspaceProfile.GetMigrationDir(), "migration.log") + + // Spinner for migration progress + sp := spinner.New( + spinner.CharSets[14], + 100*time.Millisecond, + spinner.WithHiddenCursor(true), + spinner.WithWriter(os.Stdout), + ) + // set suffix and start + sp.Suffix = " Migrating data to DuckLake format" + sp.Start() + defer sp.Stop() + + // Choose DB path for discovery + // If this is the first time we are migrating, we need to use .db file from the ~/.tailpipe/data directory + // If this is a resume migration, we need to use .db file from the ~/.tailpipe/migration/migrating directory + var discoveryDbPath string + if initialMigration { + discoveryDbPath = filepath.Join(dataDefaultDir, "tailpipe.db") + } else { + discoveryDbPath = filepath.Join(migratingDefaultDir, "tailpipe.db") + } + + sp.Suffix = " Migrating data to DuckLake format: discover tables to migrate" + // STEP 2: Discover legacy tables and their schemas (from chosen DB path) + // This returns the list of views and a map of view name to its schema + views, schemas, err := discoverLegacyTablesAndSchemas(ctx, discoveryDbPath) + if err != nil { + statusMsg = getStatusMessage(ctx, InitialisationFailed, "") + return fmt.Errorf("failed to discover legacy tables: %w", err) + } + + // STEP 3: If this is the first time we are migrating(tables in ~/.tailpipe/data) then move the whole contents of data dir + // into ~/.tailpipe/migration/migrating respecting the same folder structure. + // We do this by simply renaming the directory. + if initialMigration { + sp.Suffix = " Migrating data to DuckLake format: moving legacy data to migration area" + + if err := moveDataToMigrating(ctx, dataDefaultDir, migratingDefaultDir); err != nil { + slog.Error("Failed to move data to migrating directory", "error", err) + statusMsg = getStatusMessage(ctx, InitialisationFailed, logPath) + return err + } + } + + // STEP 4: We have now moved the data into migrating. We have the list of views from the legacy DB. + // We now need to find the matching table directories in migrating/default by scanning migrating/ + // for tp_table=* directories. + // The matching table directories are the ones that have a view in the database. + // The unmatched table directories are the ones that have data(.parquet files) but no view in the database. + // We will move these to migrated/default. + + // set the base directory to ~.tailpipe/migration/migrating/ + baseDir := migratingDefaultDir + matchedTableDirs, unmatchedTableDirs, err = findMatchingTableDirs(baseDir, views) + if err != nil { + statusMsg = getStatusMessage(ctx, MigrationFailed, logPath) + return fmt.Errorf("failed to find matching table directories: %w", err) + } + + if len(unmatchedTableDirs) > 0 { + sp.Suffix = " Migrating data to DuckLake format: archiving tables without views" + // move the unmatched table directories to 'unmigrated' + if err = archiveUnmatchedDirs(ctx, unmatchedTableDirs); err != nil { + statusMsg = getStatusMessage(ctx, MigrationFailed, logPath) + return fmt.Errorf("failed to archive unmatched table directories: %w", err) + } + } + + // Pre-compute total parquet files across matched directories + sp.Suffix = " Migrating data to DuckLake format: counting parquet files" + totalFiles, err := countParquetFiles(ctx, matchedTableDirs) + if err != nil { + return fmt.Errorf("failed to count parquet files: %w", err) + } + + // create an update func to update th espinner + updateFunc := func(st *MigrationStatus) { + sp.Suffix = fmt.Sprintf(" Migrating data to DuckLake format | tables (%d/%d) | parquet files (%d/%d, %0.1f%%)", st.MigratedTables, st.TotalTables, st.MigratedFiles, st.TotalFiles, st.ProgressPercent) + } + + // Initialize migration status, paaing in the file and table count and status update func + totalTables := len(matchedTableDirs) + status := NewMigrationStatus(totalFiles, totalTables, updateFunc) + // ensure we save the status to file at the end + defer func() { + // add any error to status and write to file before returning + if err != nil { + status.AddError(err) + if perr.IsContextCancelledError(ctx.Err()) { + // set cancel status and prune the tree + _ = onCancelled(status) + } else { + status.Finish("FAILED") + } + } + + // write the status back + _ = status.WriteStatusToFile() + }() + + // call initial update on status - this will set the spinner message correctly + status.update() + + // STEP 5: Do Migration: Traverse matched table directories, find leaf nodes with parquet files, + // and perform INSERT within a transaction. On success, move leaf dir to migrated. + err = doMigration(ctx, matchedTableDirs, schemas, status) + // If cancellation arrived during migration, prefer the CANCELLED outcome and do not + // treat it as a failure (which would incorrectly move tailpipe.db to failed) + if perr.IsContextCancelledError(ctx.Err()) { + statusMsg = getStatusMessage(ctx, MigrationFailed, logPath) + return ctx.Err() + } + + if err != nil { + statusMsg = getStatusMessage(ctx, MigrationFailed, logPath) + return fmt.Errorf("migration failed: %w", err) + } + + // Post-migration outcomes + + if status.FailedTables > 0 { + if err := onFailed(status); err != nil { + statusMsg = getStatusMessage(ctx, MigrationFailed, logPath) + return fmt.Errorf("failed to cleanup after failed migration: %w", err) + } + partialMigrated = true + statusMsg = getStatusMessage(ctx, PartialSuccess, logPath) + return err + + } + + // so we are successful - cleanup + if err := onSuccessful(status); err != nil { + statusMsg = getStatusMessage(ctx, CleanupAfterSuccess, logPath) + return fmt.Errorf("failed to cleanup after successful migration: %w", err) + } + + // all good! + statusMsg = getStatusMessage(ctx, Success, logPath) + + return err +} + +// check if the data migration is supported, based on the current arguments +// if the output for this command is a machine readable format (csv/json) or progress is false, +// it is possible/likely that tailpipe is being used in a non interactive way - in this case, +// we should not prompt the user, instead return an error +// NOTE: set exit code to +func checkMigrationSupported() error { + if error_helpers.IsMachineReadableOutput() { + return &UnsupportedError{ + Reason: "--output " + viper.GetString(constants.ArgOutput), + } + } else if viper.IsSet(constants.ArgProgress) && !viper.GetBool(constants.ArgProgress) { + return &UnsupportedError{ + Reason: "--progress=false", + } + } + return nil +} + +// moveDataToMigrating ensures the migration folder exists and handles any existing migrating folder +func moveDataToMigrating(ctx context.Context, dataDefaultDir, migratingDefaultDir string) error { + // Ensure the 'migrating' folder exists + migrationDir := config.GlobalWorkspaceProfile.GetMigratingDir() + if err := os.MkdirAll(migrationDir, 0755); err != nil { + return fmt.Errorf("failed to create migration directory: %w", err) + } + + // If the migrating folder exists, it can't have a db as we already checked - delete it + if _, err := os.Stat(migratingDefaultDir); err == nil { + // Directory exists, remove it since we already verified it doesn't contain a db + if err := os.RemoveAll(migratingDefaultDir); err != nil { + return fmt.Errorf("failed to remove existing migrating directory: %w", err) + } + } + + // Now move the data directory to the migrating directory + if err := os.Rename(dataDefaultDir, migratingDefaultDir); err != nil { + return fmt.Errorf("failed to move data to migration area: %w", err) + } + + // now recreate the moved folder + if err := os.MkdirAll(dataDefaultDir, 0755); err != nil { + return fmt.Errorf("failed to recreate data directory after moving: %w", err) + } + return nil +} + +// promptUserForMigration prompts the user to confirm migration and returns true if they want to continue +func promptUserForMigration(ctx context.Context, dataDir string) (bool, error) { + // Check if context is already cancelled + if ctx.Err() != nil { + return false, ctx.Err() + } + + //nolint: forbidigo // UI output + fmt.Printf("This version of Tailpipe requires your data to be migrated to the new Ducklake format.\n\nThis operation is irreversible. If desired, back up your data folder (%s) before proceeding.\n\nContinue? [y/N]: ", dataDir) + + // Use goroutine to read input while allowing context cancellation + type result struct { + response string + err error + } + + resultChan := make(chan result, 1) + go func() { + reader := bufio.NewReader(os.Stdin) + response, err := reader.ReadString('\n') + resultChan <- result{response, err} + }() + + select { + case <-ctx.Done(): + return false, ctx.Err() + case res := <-resultChan: + if res.err != nil { + return false, fmt.Errorf("failed to read user input: %w", res.err) + } + + response := strings.TrimSpace(strings.ToLower(res.response)) + return response == "y" || response == "yes", nil + } +} + +// getStatusMessage returns the appropriate status message based on error type and context +// It handles cancellation checking internally and returns the appropriate message +func getStatusMessage(ctx context.Context, msgType StatusType, logPath string) string { + // Handle cancellation first + if perr.IsContextCancelledError(ctx.Err()) { + switch msgType { + case InitialisationFailed: + return "Migration cancelled. Migration data cleaned up and all original data files remain unchanged. Migration will automatically resume next time you run Tailpipe.\n" + default: + return "Migration cancelled. Migration will automatically resume next time you run Tailpipe.\n" + } + } + + // Handle non-cancellation cases + switch msgType { + case InitialisationFailed: + return "Migration initialisation failed.\nMigration data cleaned up and all original data files remain unchanged. Migration will automatically resume next time you run Tailpipe.\n" + case MigrationFailed: + return fmt.Sprintf("Migration failed.\nFor details, see %s\nPlease contact Turbot support on Slack (#tailpipe).", logPath) + case CleanupAfterSuccess: + return fmt.Sprintf("Migration succeeded but cleanup failed\nFor details, see %s\n", logPath) + case PartialSuccess: + return fmt.Sprintf("Your data has been migrated to DuckLake, but some files could not be migrated.\nFor details, see %s\nIf you need help, please contact Turbot support on Slack (#tailpipe).", logPath) + // success + default: + return fmt.Sprintf("Your data has been migrated to DuckLake format.\nFor details, see %s\n", logPath) + } +} + +// discoverLegacyTablesAndSchemas enumerates legacy DuckDB views and, for each view, its schema. +// It returns the list of view names and a map of view name to its schema (column->type). +// If the legacy database contains no views, both return values are empty. +func discoverLegacyTablesAndSchemas(ctx context.Context, dbPath string) ([]string, map[string]*schema.TableSchema, error) { + // open a duckdb connection to the legacy legacyDb + legacyDb, err := database.NewDuckDb(database.WithDbFile(dbPath)) + if err != nil { + return nil, nil, err + } + defer legacyDb.Close() + + views, err := database.GetLegacyTableViews(ctx, legacyDb) + if err != nil || len(views) == 0 { + return []string{}, map[string]*schema.TableSchema{}, err + } + if perr.IsContextCancelledError(ctx.Err()) { + return nil, nil, ctx.Err() + } + + schemas := make(map[string]*schema.TableSchema) + for _, v := range views { + if perr.IsContextCancelledError(ctx.Err()) { + return nil, nil, ctx.Err() + } + // get row count for the view (optional future optimization) and schema + ts, err := database.GetLegacyTableViewSchema(ctx, v, legacyDb) + if err != nil { + continue + } + schemas[v] = ts + } + return views, schemas, nil +} + +// migrateTableDirectory recursively traverses a table directory, finds leaf nodes containing +// parquet files, and for each leaf executes a placeholder INSERT within a transaction. +// On success, it moves the leaf directory from migrating to migrated. +func migrateTableDirectory(ctx context.Context, db *database.DuckDb, tableName string, dirPath string, ts *schema.TableSchema, status *MigrationStatus) error { + // create the table if not exists + err := database.EnsureDuckLakeTable(ts.Columns, db, tableName) + if err != nil { + // fatal ā€“ move table dir to failed and return error + moveTableDirToFailed(ctx, dirPath) + return err + } + entries, err := os.ReadDir(dirPath) + if err != nil { + // fatal ā€“ move table dir to failed and return error + moveTableDirToFailed(ctx, dirPath) + return err + } + + var parquetFiles []string + aggErr := NewMigrationError() + for _, entry := range entries { + // early exit on cancellation + if ctx.Err() != nil { + aggErr.Append(ctx.Err()) + return aggErr + } + + if entry.IsDir() { + subDir := filepath.Join(dirPath, entry.Name()) + if err := migrateTableDirectory(ctx, db, tableName, subDir, ts, status); err != nil { + // just add to error list and continue with other entries + aggErr.Append(err) + } + } + + if strings.HasSuffix(strings.ToLower(entry.Name()), ".parquet") { + parquetFiles = append(parquetFiles, filepath.Join(dirPath, entry.Name())) + } + } + + // If this directory contains parquet files, treat it as a leaf node for migration + if len(parquetFiles) > 0 { + err = migrateParquetFiles(ctx, db, tableName, dirPath, ts, status, parquetFiles) + if err != nil { + aggErr.Append(err) + status.AddError(fmt.Errorf("failed migrating parquet files for table '%s' at '%s': %w", tableName, dirPath, err)) + } + } + + if aggErr.Len() == 0 { + return nil + } + return aggErr +} + +func migrateParquetFiles(ctx context.Context, db *database.DuckDb, tableName string, dirPath string, ts *schema.TableSchema, status *MigrationStatus, parquetFiles []string) error { + filesInLeaf := len(parquetFiles) + + // Begin transaction + tx, err := db.BeginTx(ctx, nil) + if err != nil { + moveTableDirToFailed(ctx, dirPath) + status.OnFilesFailed(filesInLeaf) + return err + } + + // Build and execute the parquet insert + if err := insertFromParquetFiles(ctx, tx, tableName, ts.Columns, parquetFiles); err != nil { + slog.Debug("Rolling back transaction", "table", tableName, "error", err) + txErr := tx.Rollback() + if txErr != nil { + slog.Error("Transaction rollback failed", "table", tableName, "error", txErr) + } + moveTableDirToFailed(ctx, dirPath) + status.OnFilesFailed(filesInLeaf) + return err + } + // Note: cancellation will be handled by outer logic; if needed, you can check and rollback here. + + if err := tx.Commit(); err != nil { + slog.Error("Error committing transaction", "table", tableName, "error", err) + moveTableDirToFailed(ctx, dirPath) + status.OnFilesFailed(filesInLeaf) + return err + } + + slog.Info("Successfully committed transaction", "table", tableName, "dir", dirPath, "files", filesInLeaf) + + // Clean up the now-empty source dir. If this fails (e.g., hidden files), log and continue; + // do NOT classify as a failed migration, since data has been committed successfully. + if err := os.RemoveAll(dirPath); err != nil { + slog.Warn("Cleanup: could not remove migrated leaf directory", "table", tableName, "dir", dirPath, "error", err) + } + status.OnFilesMigrated(filesInLeaf) + slog.Debug("Migrated leaf node", "table", tableName, "source", dirPath) + return nil +} + +// move any table directories with no corresponding view to ~/.tailpipe/migration/unmigrated/ - we will not migrate them +func archiveUnmatchedDirs(ctx context.Context, unmatchedTableDirs []string) error { + for _, d := range unmatchedTableDirs { + // move to ~/.tailpipe/migration/migrated/ + tname := strings.TrimPrefix(filepath.Base(d), "tp_table=") + slog.Warn("Table %s has data but no view in database; moving without migration", "table", tname, "dir", d) + migratingRoot := config.GlobalWorkspaceProfile.GetMigratingDir() + unmigratedRoot := config.GlobalWorkspaceProfile.GetUnmigratedDir() + // get the relative path from migrating root to d + rel, err := filepath.Rel(migratingRoot, d) + if err != nil { + return err + } + // build a dest path by joining unmigrated root with this relative path + destPath := filepath.Join(unmigratedRoot, rel) + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + return err + } + // move the entire directory + if err := utils.MoveDirContents(ctx, d, destPath); err != nil { + return err + } + err = os.Remove(d) + if err != nil { + return err + } + } + return nil +} + +// doMigration performs the migration of the matched table directories and updates status +func doMigration(ctx context.Context, matchedTableDirs []string, schemas map[string]*schema.TableSchema, status *MigrationStatus) error { + ducklakeDb, err := database.NewDuckDb(database.WithDuckLake()) + if err != nil { + return err + } + defer ducklakeDb.Close() + + for _, tableDir := range matchedTableDirs { + tableName := strings.TrimPrefix(filepath.Base(tableDir), "tp_table=") + if tableName == "" { + continue + } + ts := schemas[tableName] + if err := migrateTableDirectory(ctx, ducklakeDb, tableName, tableDir, ts, status); err != nil { + slog.Warn("Migration failed for table; moving to migration/failed", "table", tableName, "error", err) + status.OnTableFailed(tableName) + } else { + status.OnTableMigrated() + } + } + return nil +} + +// moveTableDirToFailed moves a table directory from migrating to failed, preserving relative path. +func moveTableDirToFailed(ctx context.Context, dirPath string) { + // If the migration was cancelled, do not classify this table as failed + if perr.IsContextCancelledError(ctx.Err()) { + return + } + migratingRoot := config.GlobalWorkspaceProfile.GetMigratingDir() + failedRoot := config.GlobalWorkspaceProfile.GetMigrationFailedDir() + rel, err := filepath.Rel(migratingRoot, dirPath) + if err != nil { + return + } + destDir := filepath.Join(failedRoot, rel) + err = os.MkdirAll(filepath.Dir(destDir), 0755) + if err != nil { + slog.Error("moveTableDirToFailed: Failed to create parent for failed dir", "error", err, "dir", destDir) + return + } + err = utils.MoveDirContents(ctx, dirPath, destDir) + if err != nil { + slog.Error("moveTableDirToFailed: Failed to move dir to failed", "error", err, "source", dirPath, "destination", destDir) + return + } + err = os.Remove(dirPath) + if err != nil { + slog.Error("moveTableDirToFailed: Failed to remove original dir after move", "error", err, "dir", dirPath) + } +} + +// countParquetFiles walks all matched table directories and counts parquet files +func countParquetFiles(ctx context.Context, dirs []string) (int, error) { + total := 0 + for _, root := range dirs { + // early exit on cancellation + if ctx.Err() != nil { + return 0, ctx.Err() + } + if err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.HasSuffix(strings.ToLower(d.Name()), ".parquet") { + total++ + } + return nil + }); err != nil { + return 0, err + } + } + return total, nil +} + +// insertFromParquetFiles builds and executes an INSERT ā€¦ SELECT read_parquet(...) for a set of parquet files +func insertFromParquetFiles(ctx context.Context, tx *sql.Tx, tableName string, columns []*schema.ColumnSchema, parquetFiles []string) error { + var colList []string + for _, c := range columns { + colList = append(colList, fmt.Sprintf(`"%s"`, c.ColumnName)) + } + cols := strings.Join(colList, ", ") + escape := func(p string) string { return strings.ReplaceAll(p, "'", "''") } + var fileSQL string + if len(parquetFiles) == 1 { + fileSQL = fmt.Sprintf("'%s'", escape(parquetFiles[0])) + } else { + var quoted []string + for _, f := range parquetFiles { + quoted = append(quoted, fmt.Sprintf("'%s'", escape(f))) + } + fileSQL = "[" + strings.Join(quoted, ", ") + "]" + } + //nolint:gosec // file paths are sanitized + query := fmt.Sprintf(` + insert into "%s" (%s) + select %s from read_parquet(%s) + `, tableName, cols, cols, fileSQL) + _, err := tx.ExecContext(ctx, query) + return err +} + +// onSuccessful handles success outcome: cleans migrating db, prunes empty dirs, prints summary +func onSuccessful(status *MigrationStatus) error { + // Remove any leftover db in migrating + if err := os.Remove(filepath.Join(config.GlobalWorkspaceProfile.GetMigratingDir(), "tailpipe.db")); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove leftover migrating db: %w", err) + } + + // Prune empty dirs in migrating + if err := filepaths.PruneTree(config.GlobalWorkspaceProfile.GetMigratingDir()); err != nil { + return fmt.Errorf("failed to prune empty directories in migrating: %w", err) + } + status.Finish("SUCCESS") + return nil +} + +// onCancelled handles cancellation outcome: keep migrating db, prune empties, print summary +func onCancelled(status *MigrationStatus) error { + // Do not move db; just prune empties so tree is clean + _ = filepaths.PruneTree(config.GlobalWorkspaceProfile.GetMigratingDir()) + status.Finish("CANCELLED") + return nil +} + +// onFailed handles failure outcome: move db to failed, prune empties, print summary +func onFailed(status *MigrationStatus) error { + status.Finish("INCOMPLETE") + + failedDefaultDir := config.GlobalWorkspaceProfile.GetMigrationFailedDir() + if err := os.MkdirAll(failedDefaultDir, 0755); err != nil { + return err + } + srcDb := filepath.Join(config.GlobalWorkspaceProfile.GetMigratingDir(), "tailpipe.db") + if _, err := os.Stat(srcDb); err == nil { + if err := utils.MoveFile(srcDb, filepath.Join(failedDefaultDir, "tailpipe.db")); err != nil { + return fmt.Errorf("failed to move legacy db to failed: %w", err) + } + } + _ = filepaths.PruneTree(config.GlobalWorkspaceProfile.GetMigratingDir()) + return nil +} diff --git a/internal/migration/status.go b/internal/migration/status.go new file mode 100644 index 00000000..1d18b96a --- /dev/null +++ b/internal/migration/status.go @@ -0,0 +1,164 @@ +package migration + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/turbot/tailpipe/internal/config" +) + +type MigrationStatus struct { + Status string `json:"status"` + TotalTables int `json:"totaltables"` + MigratedTables int `json:"migratedtables"` + FailedTables int `json:"failedtables"` + RemainingTables int `json:"remainingtables"` + ProgressPercent float64 `json:"progress_percent"` + + TotalFiles int `json:"total_files"` + MigratedFiles int `json:"migrated_files"` + FailedFiles int `json:"failed_files"` + RemainingFiles int `json:"remaining_files"` + + FailedTableNames []string `json:"failed_table_names,omitempty"` + StartTime time.Time `json:"start_time"` + Duration time.Duration `json:"duration"` + + Errors []string `json:"errors,omitempty"` + + // update func + updateFunc func(st *MigrationStatus) +} + +func NewMigrationStatus(totalFiles, totalTables int, updateFunc func(st *MigrationStatus)) *MigrationStatus { + return &MigrationStatus{ + TotalTables: totalTables, + RemainingTables: totalTables, + TotalFiles: totalFiles, + RemainingFiles: totalFiles, + StartTime: time.Now(), + updateFunc: updateFunc, + } +} + +func (s *MigrationStatus) OnTableMigrated() { + s.MigratedTables++ + s.update() +} + +func (s *MigrationStatus) OnTableFailed(tableName string) { + s.FailedTables++ + s.FailedTableNames = append(s.FailedTableNames, tableName) + s.update() +} + +func (s *MigrationStatus) OnFilesMigrated(n int) { + if n <= 0 { + return + } + s.MigratedFiles += n + s.update() +} + +func (s *MigrationStatus) OnFilesFailed(n int) { + if n <= 0 { + return + } + s.FailedFiles += n + s.update() +} + +func (s *MigrationStatus) AddError(err error) { + if err == nil { + return + } + s.Errors = append(s.Errors, err.Error()) +} + +func (s *MigrationStatus) Finish(outcome string) { + s.Status = outcome + s.Duration = time.Since(s.StartTime) +} + +// StatusMessage returns a user-facing status message (with stats) based on current migration status +func (s *MigrationStatus) StatusMessage() string { + failedDir := config.GlobalWorkspaceProfile.GetMigrationFailedDir() + migratingDir := config.GlobalWorkspaceProfile.GetMigratingDir() + + switch s.Status { + case "SUCCESS": + return fmt.Sprintf( + "DuckLake migration complete.\n"+ + "- Tables: %d/%d migrated (failed: %d, remaining: %d)\n"+ + "- Parquet files: %d/%d migrated (failed: %d, remaining: %d)\n", + s.MigratedTables, s.TotalTables, s.FailedTables, s.RemainingTables, + s.MigratedFiles, s.TotalFiles, s.FailedFiles, s.RemainingFiles, + ) + case "CANCELLED": + return fmt.Sprintf( + "DuckLake migration cancelled.\n"+ + "- Tables: %d/%d migrated (failed: %d, remaining: %d)\n"+ + "- Parquet files: %d/%d migrated (failed: %d, remaining: %d)\n"+ + "- Legacy DB preserved: '%s/tailpipe.db'\n\n"+ + "Re-run Tailpipe to resume migrating your data.\n", + s.MigratedTables, s.TotalTables, s.FailedTables, s.RemainingTables, + s.MigratedFiles, s.TotalFiles, s.FailedFiles, s.RemainingFiles, + migratingDir, + ) + case "INCOMPLETE": + failedList := "(none)" + if len(s.FailedTableNames) > 0 { + failedList = strings.Join(s.FailedTableNames, ", ") + } + base := fmt.Sprintf( + "DuckLake migration completed with issues.\n"+ + "- Tables: %d/%d migrated (failed: %d, remaining: %d)\n"+ + "- Parquet files: %d/%d migrated (failed: %d, remaining: %d)\n"+ + "- Failed tables (%d): %s\n"+ + "- Failed data and legacy DB: '%s'\n", + s.MigratedTables, s.TotalTables, s.FailedTables, s.RemainingTables, + s.MigratedFiles, s.TotalFiles, s.FailedFiles, s.RemainingFiles, + len(s.FailedTableNames), failedList, + failedDir, + ) + if len(s.Errors) > 0 { + base += fmt.Sprintf("\nErrors: %d error(s) occurred during migration\n", len(s.Errors)) + base += "Details:\n" + for _, e := range s.Errors { + base += "- " + e + "\n" + } + } + return base + default: + return "DuckLake migration status unknown" + } +} + +// WriteStatusToFile writes the status message to a migration stats file under the migration directory. +// The file is overwritten on each run (resume will update it). +func (s *MigrationStatus) WriteStatusToFile() error { + // Place the file under the migration root (e.g., ~/.tailpipe/migration/migration.log) + migrationRootDir := config.GlobalWorkspaceProfile.GetMigrationDir() + statsFile := filepath.Join(migrationRootDir, "migration.log") + msg := s.StatusMessage() + if err := os.MkdirAll(migrationRootDir, 0755); err != nil { + return err + } + return os.WriteFile(statsFile, []byte(msg), 0600) +} + +// update recalculates remaining counts and progress percent, and calls the update func if set +func (s *MigrationStatus) update() { + s.RemainingTables = s.TotalTables - s.MigratedTables - s.FailedTables + s.RemainingFiles = s.TotalFiles - s.MigratedFiles - s.FailedFiles + if s.TotalFiles > 0 { + s.ProgressPercent = float64(s.MigratedFiles+s.FailedFiles) * 100.0 / float64(s.TotalFiles) + } + // call our update func + if s.updateFunc != nil { + s.updateFunc(s) + } +} diff --git a/internal/migration/utils.go b/internal/migration/utils.go new file mode 100644 index 00000000..7cf485e5 --- /dev/null +++ b/internal/migration/utils.go @@ -0,0 +1,50 @@ +package migration + +import ( + "os" + "path/filepath" + "strings" +) + +// findMatchingTableDirs lists subdirectories of baseDir whose names start with +// "tp_table=" and whose table names exist in the provided tables slice. +// Also returns unmatched tp_table directories for which there is no view in the DB. +func findMatchingTableDirs(baseDir string, tables []string) ([]string, []string, error) { + entries, err := os.ReadDir(baseDir) + if err != nil { + return nil, nil, err + } + tableSet := make(map[string]struct{}, len(tables)) + for _, t := range tables { + tableSet[t] = struct{}{} + } + var matches []string + var unmatched []string + const prefix = "tp_table=" + for _, e := range entries { + if !e.IsDir() { + continue + } + name := e.Name() + if !strings.HasPrefix(name, prefix) { + continue + } + tableName := strings.TrimPrefix(name, prefix) + if _, ok := tableSet[tableName]; ok { + matches = append(matches, filepath.Join(baseDir, name)) + } else { + unmatched = append(unmatched, filepath.Join(baseDir, name)) + } + } + return matches, unmatched, nil +} + +// hasTailpipeDb checks if a tailpipe.db file exists in the provided directory. +func hasTailpipeDb(dir string) bool { + if dir == "" { + return false + } + p := filepath.Join(dir, "tailpipe.db") + _, err := os.Stat(p) + return err == nil +} diff --git a/internal/parquet/compact.go b/internal/parquet/compact.go deleted file mode 100644 index 11c8bd94..00000000 --- a/internal/parquet/compact.go +++ /dev/null @@ -1,264 +0,0 @@ -package parquet - -import ( - "context" - "fmt" - "golang.org/x/sync/semaphore" - "log/slog" - "os" - "path/filepath" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/turbot/pipe-fittings/v2/utils" - "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/database" -) - -type CompactionStatus struct { - Source int - Dest int - Uncompacted int -} - -func (total *CompactionStatus) Update(counts CompactionStatus) { - total.Source += counts.Source - total.Dest += counts.Dest - total.Uncompacted += counts.Uncompacted -} - -func (total *CompactionStatus) VerboseString() string { - if total.Source == 0 && total.Dest == 0 && total.Uncompacted == 0 { - return "No files to compact." - } - - uncompactedString := "" - if total.Uncompacted > 0 { - uncompactedString = fmt.Sprintf("%d files did not need compaction.", total.Uncompacted) - } - - // if nothing was compacated, just print the uncompacted string - if total.Source == 0 { - return fmt.Sprintf("%s\n\n", uncompactedString) - } - - // add brackets around the uncompacted string (if needed) - if len(uncompactedString) > 0 { - uncompactedString = fmt.Sprintf(" (%s)", uncompactedString) - } - - return fmt.Sprintf("Compacted %d files into %d files.%s\n", total.Source, total.Dest, uncompactedString) -} - -func (total *CompactionStatus) BriefString() string { - if total.Source == 0 { - return "" - } - - uncompactedString := "" - if total.Uncompacted > 0 { - uncompactedString = fmt.Sprintf(" (%d files did not need compaction.)", total.Uncompacted) - } - - return fmt.Sprintf("Compacted %d files into %d files.%s\n", total.Source, total.Dest, uncompactedString) -} - -func CompactDataFiles(ctx context.Context, updateFunc func(CompactionStatus), patterns ...PartitionPattern) error { - // get the root data directory - baseDir := config.GlobalWorkspaceProfile.GetDataDir() - - // open a duckdb connection - db, err := database.NewDuckDb() - if err != nil { - return fmt.Errorf("failed to open duckdb connection: %w", err) - } - defer db.Close() - - // traverse the directory and compact files - if err := traverseAndCompact(ctx, db, baseDir, updateFunc, patterns); err != nil { - return err - } - invalidDeleteErr := deleteInvalidParquetFiles(config.GlobalWorkspaceProfile.GetDataDir(), patterns) - if invalidDeleteErr != nil { - slog.Warn("Failed to delete invalid parquet files", "error", invalidDeleteErr) - } - return nil -} -func traverseAndCompact(ctx context.Context, db *database.DuckDb, dirPath string, updateFunc func(CompactionStatus), patterns []PartitionPattern) error { - // if this is the partition folder, check if it matches the patterns before descending further - if table, partition, ok := getPartitionFromPath(dirPath); ok { - if !PartitionMatchesPatterns(table, partition, patterns) { - return nil - } - } - - entries, err := os.ReadDir(dirPath) - if err != nil { - return fmt.Errorf("failed to read directory %s: %w", dirPath, err) - } - - var parquetFiles []string - - // process directory entries - for _, entry := range entries { - if entry.IsDir() { - // recursively process subdirectories - subDirPath := filepath.Join(dirPath, entry.Name()) - err := traverseAndCompact(ctx, db, subDirPath, updateFunc, patterns) - if err != nil { - return err - } - } else if strings.HasSuffix(entry.Name(), ".parquet") { - // collect parquet file paths - parquetFiles = append(parquetFiles, filepath.Join(dirPath, entry.Name())) - } - } - numFiles := len(parquetFiles) - if numFiles < 2 { - // nothing to compact - update the totals anyway so we include uncompacted files in the overall total - updateFunc(CompactionStatus{Uncompacted: numFiles}) - return nil - } - - err = compactParquetFiles(ctx, db, parquetFiles, dirPath) - if err != nil { - if ctx.Err() != nil { - return err - } - return fmt.Errorf("failed to compact parquet files in %s: %w", dirPath, err) - } - - // update the totals - updateFunc(CompactionStatus{Source: numFiles, Dest: 1}) - - return nil -} - -// if this parquetFile ends with the partition segment, return the table and partition -func getPartitionFromPath(dirPath string) (string, string, bool) { - // if this is a partition folder, check if it matches the patterns - parts := strings.Split(dirPath, "/") - l := len(parts) - if l > 1 && strings.HasPrefix(parts[l-1], "tp_partition=") && strings.HasPrefix(parts[l-2], "tp_table=") { - - table := strings.TrimPrefix(parts[l-2], "tp_table=") - partition := strings.TrimPrefix(parts[l-1], "tp_partition=") - return table, partition, true - } - return "", "", false -} - -func compactParquetFiles(ctx context.Context, db *database.DuckDb, parquetFiles []string, inputPath string) (err error) { - now := time.Now() - compactedFileName := fmt.Sprintf("snap_%s_%06d.parquet", now.Format("20060102150405"), now.Nanosecond()/1000) - - if !filepath.IsAbs(inputPath) { - return fmt.Errorf("inputPath must be an absolute path") - } - // define temp and output file paths - tempOutputFile := filepath.Join(inputPath, compactedFileName+".tmp") - outputFile := filepath.Join(inputPath, compactedFileName) - - defer func() { - if err != nil { - if ctx.Err() == nil { - slog.Error("Compaction failed", "inputPath", inputPath, "error", err) - } - // delete temp file if it exists - _ = os.Remove(tempOutputFile) - } - }() - - // compact files using duckdb - query := fmt.Sprintf(` - copy ( - select * from read_parquet('%s/*.parquet') - ) to '%s' (format parquet, overwrite true); - `, inputPath, tempOutputFile) - - if _, err := db.ExecContext(ctx, query); err != nil { - if ctx.Err() != nil { - return err - } - return fmt.Errorf("failed to compact parquet files: %w", err) - } - - // rename all parquet files to add a .compacted extension - err = renameCompactedFiles(parquetFiles) - if err != nil { - // delete the temp file - _ = os.Remove(tempOutputFile) - return err - } - - // rename temp file to final output file - if err := os.Rename(tempOutputFile, outputFile); err != nil { - return fmt.Errorf("failed to rename temp file %s to %s: %w", tempOutputFile, outputFile, err) - } - - // finally, delete renamed source parquet files - err = deleteCompactedFiles(ctx, parquetFiles) - - return nil -} - -// renameCompactedFiles renames all parquet files to add a .compacted extension -func renameCompactedFiles(parquetFiles []string) error { - - var renamedFiles []string - for _, file := range parquetFiles { - newFile := file + ".compacted" - renamedFiles = append(renamedFiles, newFile) - if err := os.Rename(file, newFile); err != nil { - // try to rename all files we have already renamed back to their original names - for _, renamedFile := range renamedFiles { - // remove the .compacted extension - originalFile := strings.TrimSuffix(renamedFile, ".compacted") - if err := os.Rename(renamedFile, originalFile); err != nil { - slog.Warn("Failed to rename parquet file back to original name", "file", renamedFile, "error", err) - } - } - return fmt.Errorf("failed to rename parquet file %s to %s: %w", file, newFile, err) - } - } - return nil -} - -// deleteCompactedFiles deletes all parquet files with a .compacted extension -func deleteCompactedFiles(ctx context.Context, parquetFiles []string) error { - const maxConcurrentDeletions = 5 - sem := semaphore.NewWeighted(int64(maxConcurrentDeletions)) - var wg sync.WaitGroup - var failures int32 - - for _, file := range parquetFiles { - wg.Add(1) - go func(file string) { - defer wg.Done() - - // Acquire a slot in the semaphore - if err := sem.Acquire(ctx, 1); err != nil { - atomic.AddInt32(&failures, 1) - return - } - defer sem.Release(1) - - newFile := file + ".compacted" - if err := os.Remove(newFile); err != nil { - atomic.AddInt32(&failures, 1) - } - }(file) - } - - // Wait for all deletions to complete - wg.Wait() - - // Check failure count atomically - failureCount := atomic.LoadInt32(&failures) - if failureCount > 0 { - return fmt.Errorf("failed to delete %d parquet %s", failureCount, utils.Pluralize("file", int(failureCount))) - } - return nil -} diff --git a/internal/parquet/conversion_error.go b/internal/parquet/conversion_error.go deleted file mode 100644 index 987eecb7..00000000 --- a/internal/parquet/conversion_error.go +++ /dev/null @@ -1,84 +0,0 @@ -package parquet - -import ( - "bytes" - "errors" - "fmt" - "io" - "log/slog" - "os" - "path/filepath" - "strings" -) - -// handleConversionError attempts to handle conversion errors by counting the number of lines in the file. -// if we fail, just return the raw error. -func handleConversionError(err error, path string) error { - logArgs := []any{ - "error", - err, - "path", - path, - } - - // try to count the number of rows in the file - rows, countErr := countLines(path) - if countErr == nil { - logArgs = append(logArgs, "rows_affected", rows) - } - - // log error - slog.Error("parquet conversion failed", logArgs...) - - // return wrapped error - return NewConversionError(err.Error(), rows, path) -} - -func countLines(filename string) (int64, error) { - file, err := os.Open(filename) - if err != nil { - return 0, err - } - defer file.Close() - - buf := make([]byte, 64*1024) - count := 0 - - for { - c, err := file.Read(buf) - if c > 0 { - count += bytes.Count(buf[:c], []byte{'\n'}) - } - if err != nil { - if err == io.EOF { - return int64(count), nil - } - return 0, err - } - } -} - -type ConversionError struct { - SourceFile string - BaseError error - RowsAffected int64 - displayError string -} - -func NewConversionError(msg string, rowsAffected int64, path string) *ConversionError { - return &ConversionError{ - SourceFile: filepath.Base(path), - BaseError: errors.New(msg), - RowsAffected: rowsAffected, - displayError: strings.Split(msg, "\n")[0], - } -} - -func (c *ConversionError) Error() string { - return fmt.Sprintf("%s: %s", c.SourceFile, c.displayError) -} - -// Merge adds a second error to the conversion error message. -func (c *ConversionError) Merge(err error) { - c.BaseError = fmt.Errorf("%s\n%s", c.BaseError.Error(), err.Error()) -} diff --git a/internal/parquet/conversion_worker.go b/internal/parquet/conversion_worker.go deleted file mode 100644 index 2c8d983e..00000000 --- a/internal/parquet/conversion_worker.go +++ /dev/null @@ -1,438 +0,0 @@ -package parquet - -import ( - "context" - "errors" - "fmt" - "log/slog" - "os" - "path/filepath" - "strings" - "sync/atomic" - "time" - - sdkconstants "github.com/turbot/tailpipe-plugin-sdk/constants" - "github.com/turbot/tailpipe-plugin-sdk/table" - "github.com/turbot/tailpipe/internal/constants" - "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/filepaths" -) - -type parquetJob struct { - chunkNumber int32 -} - -// conversionWorker is an implementation of worker that converts JSONL files to Parquet -type conversionWorker struct { - // channel to receive jobs from the writer - jobChan chan *parquetJob - - // the parent converter - converter *Converter - - // source file location - sourceDir string - // dest file location - destDir string - - // helper struct which provides unique filename roots - fileRootProvider *FileRootProvider - db *database.DuckDb -} - -func newParquetConversionWorker(converter *Converter) (*conversionWorker, error) { - w := &conversionWorker{ - jobChan: converter.jobChan, - sourceDir: converter.sourceDir, - destDir: converter.destDir, - fileRootProvider: converter.fileRootProvider, - converter: converter, - } - - // create a new DuckDB instance for each worker - db, err := database.NewDuckDb(database.WithDuckDbExtensions(constants.DuckDbExtensions)) - if err != nil { - return nil, fmt.Errorf("failed to create DuckDB wrapper: %w", err) - } - w.db = db - return w, nil -} - -// this is the worker function run by all workers, which all read from the ParquetJobPool channel -func (w *conversionWorker) start(ctx context.Context) { - slog.Debug("worker start") - // this function runs as long as the worker is running - - // ensure to close on exit - defer w.close() - - // loop until we are closed - for { - select { - case <-ctx.Done(): - // we are done - return - case job := <-w.jobChan: - if job == nil { - // we are done - return - } - if err := w.doJSONToParquetConversion(job.chunkNumber); err != nil { - // send the error to the converter - w.converter.addJobErrors(err) - continue - } - // atomically increment the completion count on our converter - atomic.AddInt32(&w.converter.completionCount, 1) - - } - } -} - -func (w *conversionWorker) close() { - _ = w.db.Close() -} - -func (w *conversionWorker) doJSONToParquetConversion(chunkNumber int32) error { - // ensure we signal the converter when we are done - defer w.converter.wg.Done() - startTime := time.Now() - - // build the source filename - jsonFileName := table.ExecutionIdToJsonlFileName(w.converter.id, chunkNumber) - jsonFilePath := filepath.Join(w.sourceDir, jsonFileName) - - // process the ParquetJobPool - rowCount, err := w.convertFile(jsonFilePath) - // if we have an error, return it below - // update the row count - w.converter.updateRowCount(rowCount) - - // delete JSON file (configurable?) - if removeErr := os.Remove(jsonFilePath); removeErr != nil { - // log the error but don't fail - slog.Error("failed to delete JSONL file", "file", jsonFilePath, "error", removeErr) - } - activeDuration := time.Since(startTime) - slog.Debug("converted JSONL to Parquet", "file", jsonFilePath, "duration (ms)", activeDuration.Milliseconds()) - // remove the conversion error (if any) - return err -} - -// convert the given jsonl file to parquet -func (w *conversionWorker) convertFile(jsonlFilePath string) (_ int64, err error) { - partition := w.converter.Partition - - // verify the jsonl file has a .jsonl extension - if filepath.Ext(jsonlFilePath) != ".jsonl" { - return 0, NewConversionError("invalid file type - conversionWorker only supports .jsonl files", 0, jsonlFilePath) - } - // verify file exists - if _, err := os.Stat(jsonlFilePath); os.IsNotExist(err) { - return 0, NewConversionError("file does not exist", 0, jsonlFilePath) - } - - // render the view query - selectQuery := fmt.Sprintf(w.converter.viewQueryFormat, jsonlFilePath) - - // if the partition includes a filter, add a where clause - if partition.Filter != "" { - selectQuery += fmt.Sprintf(" where %s", partition.Filter) - } - - // we need to perform validation if: - // 1. the table schema was partial (i.e. this is a custom table with RowMappings defined) - // 2. this is a directly converted artifact - // 3. any required columns have transforms - - // (NOTE: if we validate we write the data to a temp table then update the select query to read from that temp table) - // get list of columns to validate - columnsToValidate := w.getColumnsToValidate() - - var cleanupQuery string - var rowValidationError error - - selectQuery, cleanupQuery, rowValidationError = w.validateSchema(jsonlFilePath, selectQuery, columnsToValidate) - - if cleanupQuery != "" { - defer func() { - // TODO benchmark whether dropping the table actually makes any difference to memory pressure - // or can we rely on the drop if exists? - // validateSchema creates the table temp_data - the cleanupQuery drops it - _, tempTableError := w.db.Exec(cleanupQuery) - if tempTableError != nil { - slog.Error("failed to drop temp table", "error", err) - // if we do not already have an error return this error - if err == nil { - err = tempTableError - } - } - }() - } - // did validation fail - if rowValidationError != nil { - // if no select query was returned, we cannot proceed - just return the error - if selectQuery == "" { - return 0, rowValidationError - } - // so we have a validation error - but we DID return an updated select query - that indicates we should - // continue and just report the row validation errors - - // ensure that we return the row validation error, merged with any other error we receive - defer func() { - if err == nil { - err = rowValidationError - } else { - var conversionError *ConversionError - if errors.As(rowValidationError, &conversionError) { - // we have a conversion error - we need to set the row count to 0 - // so we can report the error - conversionError.Merge(err) - } - err = conversionError - } - }() - } - - // Create a query to write to partitioned parquet files - - // get a unique file root - fileRoot := w.fileRootProvider.GetFileRoot() - - // build a query to export the rows to partitioned parquet files - // NOTE: we initially write to a file with the extension '.parquet.tmp' - this is to avoid the creation of invalid parquet files - // in the case of a failure - // once the conversion is complete we will rename them - partitionColumns := []string{sdkconstants.TpTable, sdkconstants.TpPartition, sdkconstants.TpIndex, sdkconstants.TpDate} - exportQuery := fmt.Sprintf(`copy (%s) to '%s' ( - format parquet, - partition_by (%s), - return_files true, - overwrite_or_ignore, - filename_pattern '%s_{i}', - file_extension '%s' -);`, - selectQuery, - w.destDir, - strings.Join(partitionColumns, ","), - fileRoot, - strings.TrimPrefix(filepaths.TempParquetExtension, "."), // no '.' required for duckdb - ) - - row := w.db.QueryRow(exportQuery) - var rowCount int64 - var files []interface{} - err = row.Scan(&rowCount, &files) - if err != nil { - // try to get the row count of the file we failed to convert - return 0, handleConversionError(err, jsonlFilePath) - - } - slog.Debug("created parquet files", "count", len(files)) - - // now rename the parquet files - err = w.renameTempParquetFiles(files) - - return rowCount, err -} - -// getColumnsToValidate returns the list of columns which need to be validated at this staghe -// normally, required columns are validated in the plugin, however there are a couple of exceptions: -// 1. if the format is one which supports direct artifact-JSONL conversion (i.e. jsonl, delimited) we must validate all required columns -// 2. if any required columns have transforms, we must validate those columns as the transforms are applied at this stage -// TODO once we have tested/benchmarked, move all validation here and do not validate in plugin even for static tables -func (w *conversionWorker) getColumnsToValidate() []string { - var res []string - // if the format is one which requires a direct conversion we must validate all required columns - formatSupportsDirectConversion := w.converter.Partition.FormatSupportsDirectConversion() - - // otherwise validate required columns which have a transform - for _, col := range w.converter.conversionSchema.Columns { - if col.Required && (col.Transform != "" || formatSupportsDirectConversion) { - res = append(res, col.ColumnName) - } - } - return res -} - -// validateSchema copies the data from the given select query to a temp table and validates required fields are non null -// it also validates that the schema of the chunk is the same as the inferred schema and if it is not, reports a useful error -// the query count of invalid rows and a list of null fields -func (w *conversionWorker) validateSchema(jsonlFilePath string, selectQuery string, columnsToValidate []string) (string, string, error) { - // if we have no columns to validate, just write the data to the temp table, - // a process which will fail if the schema has changed, alerting us of this error - if len(columnsToValidate) == 0 { - return w.schemasChangeValidation(jsonlFilePath, selectQuery) - } - - // if we have no columns to validate, biuld a validation query to return the number of invalid rows and the columns with nulls - validationQuery := w.buildValidationQuery(selectQuery, columnsToValidate) - - row := w.db.QueryRow(validationQuery) - var totalRows int64 - var columnsWithNullsInterface []interface{} - - err := row.Scan(&totalRows, &columnsWithNullsInterface) - if err != nil { - - // just return the original error - return "", "", w.handleSchemaChangeError(err, jsonlFilePath) - } - - // Convert the interface slice to string slice - var columnsWithNulls []string - for _, col := range columnsWithNullsInterface { - if col != nil { - columnsWithNulls = append(columnsWithNulls, col.(string)) - } - } - - var rowValidationError error - if totalRows > 0 { - // we have a failure - return an error with details about which columns had nulls - nullColumns := "unknown" - if len(columnsWithNulls) > 0 { - nullColumns = strings.Join(columnsWithNulls, ", ") - } - // if we have row validation errors, we return a ConversionError but also return the select query - // so the calling code can convert the rest of the chunk - rowValidationError = NewConversionError(fmt.Sprintf("validation failed - found null values in columns: %s", nullColumns), totalRows, jsonlFilePath) - - // delete invalid rows from the temp table - if err := w.deleteInvalidRows(columnsToValidate); err != nil { - return "", "", handleConversionError(err, jsonlFilePath) - } - } - // build a select query to select the data back out of the temp table (excluding rowid, which we added for validation) - selectQuery = "select * exclude rowid from temp_data" - cleanupQuery := "drop table temp_data" - return selectQuery, cleanupQuery, rowValidationError -} - -func (w *conversionWorker) schemasChangeValidation(jsonlFilePath string, selectQuery string) (string, string, error) { - var queryBuilder strings.Builder - // Step 1: Create a temporary table to hold the data we want to validate - // This allows us to efficiently check multiple columns without scanning the source multiple times - queryBuilder.WriteString("drop table if exists temp_data;\n") - // note: extra select wrapper is used to allow for wrapping query before filter is applied so filter can use struct fields with dot-notation - queryBuilder.WriteString(fmt.Sprintf("create temp table temp_data as select * from (%s);\n", selectQuery)) - _, err := w.db.Exec(queryBuilder.String()) - if err != nil { - return "", "", w.handleSchemaChangeError(err, jsonlFilePath) - - } - selectQuery = "select * from temp_data" - cleanupQuery := "drop table temp_data" - - return selectQuery, cleanupQuery, nil -} - -// handleSchemaChangeError determines if the error is because the schema of this chunk is different to the inferred schema -// infer the schema of this chunk and compare - if they are different, return that in an error -func (w *conversionWorker) handleSchemaChangeError(err error, jsonlFilePath string) error { - schemaChangeErr := w.converter.detectSchemaChange(jsonlFilePath) - if schemaChangeErr != nil { - // if the error returned is a SchemaChangeError, we can return that so update the val of err - var e = &SchemaChangeError{} - if errors.As(schemaChangeErr, &e) { - // try to get the row count of the file we failed to convert - return handleConversionError(e, jsonlFilePath) - } - } - - // just return the original error - return handleConversionError(err, jsonlFilePath) -} - -// buildValidationQuery builds a query to copy the data from the select query to a temp table -// it then validates that the required columns are not null, removing invalid rows and returning -// the count of invalid rows and the columns with nulls -func (w *conversionWorker) buildValidationQuery(selectQuery string, columnsToValidate []string) string { - queryBuilder := strings.Builder{} - - // Step 1: Create a temporary table to hold the data we want to validate - // This allows us to efficiently check multiple columns without scanning the source multiple times - // NOTE: add in row number to allow for distinct row counting - we must exclude this when we select data out of the temp table again - queryBuilder.WriteString("drop table if exists temp_data;\n") - queryBuilder.WriteString(fmt.Sprintf("create temp table temp_data as select row_number() over () as rowid, * from (%s);\n", selectQuery)) - - // Step 2: Build the validation query that: - // - Counts distinct rows that have null values in required columns - // - Lists all required columns that contain null values - queryBuilder.WriteString(`select - count(distinct rowid) as rows_with_required_nulls, -- Count unique rows with nulls in required columns - coalesce(list(distinct col), []) as required_columns_with_nulls -- List required columns that have null values, defaulting to empty list if NULL -from (`) - - // Step 3: For each required column we need to validate: - // - Create a query that selects rows where this column is null - // - Include the column name so we know which column had the null - // - UNION ALL combines all these results (faster than UNION as we don't need to deduplicate) - for i, col := range columnsToValidate { - if i > 0 { - queryBuilder.WriteString(" union all\n") - } - // For each required column, create a query that: - // - Selects the rowid (to count distinct rows) - // - Includes the column name (to list which columns had nulls) - // - Only includes rows where this column is null - queryBuilder.WriteString(fmt.Sprintf(" select rowid, '%s' as col from temp_data where %s is null\n", col, col)) - } - - queryBuilder.WriteString(");") - - return queryBuilder.String() -} - -// buildNullCheckQuery builds a WHERE clause to check for null values in the specified columns -func (w *conversionWorker) buildNullCheckQuery(columnsToValidate []string) string { - if len(columnsToValidate) == 0 { - return "" - } - - // build a slice of null check conditions - conditions := make([]string, len(columnsToValidate)) - for i, col := range columnsToValidate { - conditions[i] = fmt.Sprintf("%s is null", col) - } - return strings.Join(conditions, " or ") -} - -// deleteInvalidRows removes rows with null values in the specified columns from the temp table -func (w *conversionWorker) deleteInvalidRows(columnsToValidate []string) error { - if len(columnsToValidate) == 0 { - return nil - } - - whereClause := w.buildNullCheckQuery(columnsToValidate) - query := fmt.Sprintf("delete from temp_data where %s;", whereClause) - - _, err := w.db.Exec(query) - return err -} - -// renameTempParquetFiles renames the given list of temporary parquet files to have a .parquet extension. -// note: we receive the list of files as an interface{} as that is what we read back from the db -func (w *conversionWorker) renameTempParquetFiles(files []interface{}) error { - var errList []error - for _, f := range files { - fileName := f.(string) - if strings.HasSuffix(fileName, filepaths.TempParquetExtension) { - newName := strings.TrimSuffix(fileName, filepaths.TempParquetExtension) + ".parquet" - if err := os.Rename(fileName, newName); err != nil { - errList = append(errList, fmt.Errorf("%s: %w", fileName, err)) - } - } - } - - if len(errList) > 0 { - var msg strings.Builder - msg.WriteString(fmt.Sprintf("Failed to rename %d parquet files:\n", len(errList))) - for _, err := range errList { - msg.WriteString(fmt.Sprintf(" - %v\n", err)) - } - return errors.New(msg.String()) - } - - return nil -} diff --git a/internal/parquet/conversion_worker_test.go b/internal/parquet/conversion_worker_test.go deleted file mode 100644 index 80dddc4f..00000000 --- a/internal/parquet/conversion_worker_test.go +++ /dev/null @@ -1,1523 +0,0 @@ -package parquet - -import ( - _ "github.com/marcboeker/go-duckdb/v2" -) - -//var testDb *database.DuckDb -// -//const testDir = "buildViewQuery_test_data" -// -//// we use the same path for all tests -//var jsonlFilePath string -// -//func setup() error { -// var err error -// -// // Create a temporary config directory -// tempConfigDir, err := os.MkdirTemp("", "tailpipe_test_config") -// if err != nil { -// return fmt.Errorf("error creating temp config directory: %w", err) -// } -// -// // Set the config path to our temporary directory -// viper.Set("config_path", tempConfigDir) -// -// // Initialize workspace profile with parse options -// parseOpts := []parse.ParseHclOpt{ -// parse.WithEscapeBackticks(true), -// parse.WithDisableTemplateForProperties(constants.GrokConfigProperties), -// } -// loader, err := pcmdconfig.GetWorkspaceProfileLoader[*workspace_profile.TailpipeWorkspaceProfile](parseOpts...) -// if err != nil { -// return fmt.Errorf("error creating workspace profile loader: %w", err) -// } -// config.GlobalWorkspaceProfile = loader.GetActiveWorkspaceProfile() -// if err := config.GlobalWorkspaceProfile.EnsureWorkspaceDirs(); err != nil { -// return fmt.Errorf("error ensuring workspace dirs: %w", err) -// } -// -// db, err := database.NewDuckDb(database.WithDuckDbExtensions(constants.DuckDbExtensions)) -// if err != nil { -// return fmt.Errorf("error creating duckdb: %w", err) -// } -// testDb = db -// // make tempdata directory in local folder -// // Create the directory -// err = os.MkdirAll(testDir, 0755) -// if err != nil { -// db.Close() -// return fmt.Errorf("error creating temp directory: %w", err) -// } -// -// // resolve the jsonl file path -// jsonlFilePath, err = filepath.Abs(filepath.Join(testDir, "test.jsonl")) -// return err -//} -// -//func teardown() { -// os.RemoveAll("test_data") -// if testDb != nil { -// testDb.Close() -// } -//} - -// TODO KAI FIX -//func TestBuildViewQuery(t *testing.T) { -// // set the version explicitly here since version is set during build time -// // then set the app specific constants needed for the tests -// viper.Set("main.version", "0.0.1") -// cmdconfig.SetAppSpecificConstants() -// -// if err := setup(); err != nil { -// t.Fatalf("error setting up test: %s", err) -// } -// defer teardown() -// -// type args struct { -// schema *schema.ConversionSchema -// json string -// sqlColumn string -// } -// tests := []struct { -// name string -// args args -// wantQuery string -// wantData any -// }{ -// /* -// c.Type = "boolean" -// c.Type = "tinyint" -// c.Type = "smallint" -// c.Type = "integer" -// c.Type = "bigint" -// c.Type = "utinyint" -// c.Type = "usmallint" -// c.Type = "uinteger" -// c.Type = "ubigint" -// c.Type = "float" -// c.Type = "double" -// c.Type = "varchar" -// c.Type = "timestamp" -// -// c.Type = "blob" -// c.Type = "array" -// c.Type = "struct" -// c.Type = "map" -// */ -// { -// name: "struct", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructField", -// ColumnName: "struct_field", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "varchar"}, -// {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "bigint"}, -// }, -// }, -// }, -// }, -// }, -// json: `{ "StructField": { "StructStringField": "StructStringVal", "StructIntField": 100 }}`, -// sqlColumn: "struct_field.struct_string_field", -// }, -// wantQuery: `select * from (select -// case -// when "StructField" is null then null -// else struct_pack( -// "struct_string_field" := "StructField"."StructStringField"::varchar, -// "struct_int_field" := "StructField"."StructIntField"::bigint -// ) -// end as "struct_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "StructField": 'struct("StructStringField" varchar, "StructIntField" bigint)' -// } -// ))`, -// wantData: []any{"StructStringVal"}, -// }, -// { -// name: "json", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "JsonField", -// ColumnName: "json_field", -// Type: "json", -// }, -// }, -// }, -// }, -// json: `{ "JsonField": { "string_field": "JsonStringVal", "int_field": 100 }}`, -// sqlColumn: "json_field.string_field", -// }, -// wantQuery: `select * from (select -// json("JsonField") as "json_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "JsonField": 'json' -// } -// ))`, -// wantData: []any{`JsonStringVal`}, -// }, -// { -// name: "struct with keyword names", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "end", -// ColumnName: "end", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "any", ColumnName: "any", Type: "varchar"}, -// }, -// }, -// }, -// }, -// }, -// json: `{ "end": { "any": "StructStringVal" }}`, -// sqlColumn: `"end"."any"`, -// }, -// wantQuery: `select * from (select -// case -// when "end" is null then null -// else struct_pack( -// "any" := "end"."any"::varchar -// ) -// end as "end" -//from -// read_ndjson( -// '%s', -// columns = { -// "end": 'struct("any" varchar)' -// } -// ))`, -// wantData: []any{"StructStringVal"}, -// }, -// { -// name: "null struct", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "end", -// ColumnName: "end", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "any", ColumnName: "any", Type: "varchar"}, -// }, -// }, -// }, -// }, -// }, -// json: `{ }`, -// sqlColumn: `"end"."any"`, -// }, -// wantQuery: `select * from (select -// case -// when "end" is null then null -// else struct_pack( -// "any" := "end"."any"::varchar -// ) -// end as "end" -//from -// read_ndjson( -// '%s', -// columns = { -// "end": 'struct("any" varchar)' -// } -// ))`, -// wantData: []any{nil}, -// }, -// { -// name: "nested struct", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructField", -// ColumnName: "struct_field", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// { -// SourceName: "NestedStruct", -// ColumnName: "nested_struct", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// { -// SourceName: "NestedStructStringField", -// ColumnName: "nested_struct_string_field", -// Type: "varchar", -// }, -// }, -// }, -// { -// SourceName: "StructStringField", -// ColumnName: "struct_string_field", -// Type: "varchar", -// }, -// }, -// }, -// }, -// }, -// }, -// json: `{ "StructField": { "NestedStruct": { "NestedStructStringField": "NestedStructStringVal" }, "StructStringField": "StructStringVal" }}`, -// sqlColumn: "struct_field.nested_struct.nested_struct_string_field", -// }, -// wantQuery: `select * from (select -// case -// when "StructField" is null then null -// else struct_pack( -// "nested_struct" := case -// when "StructField"."NestedStruct" is null then null -// else struct_pack( -// "nested_struct_string_field" := "StructField"."NestedStruct"."NestedStructStringField"::varchar -// ) -// end, -// "struct_string_field" := "StructField"."StructStringField"::varchar -// ) -// end as "struct_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "StructField": 'struct("NestedStruct" struct("NestedStructStringField" varchar), "StructStringField" varchar)' -// } -// ))`, -// wantData: []any{"NestedStructStringVal"}, -// }, -// { -// name: "null nested struct", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructField", -// ColumnName: "struct_field", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// { -// SourceName: "NestedStruct", -// ColumnName: "nested_struct", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// { -// SourceName: "NestedStructStringField", -// ColumnName: "nested_struct_string_field", -// Type: "varchar", -// }, -// }, -// }, -// { -// SourceName: "StructStringField", -// ColumnName: "struct_string_field", -// Type: "varchar", -// }, -// }, -// }, -// }, -// }, -// }, -// json: `{ "StructField": { "NestedStruct": { "NestedStructStringField": "NestedStructStringVal" }, "StructStringField": "StructStringVal" }} -//{ }`, -// sqlColumn: "struct_field.nested_struct.nested_struct_string_field", -// }, -// wantQuery: `select * from (select -// case -// when "StructField" is null then null -// else struct_pack( -// "nested_struct" := case -// when "StructField"."NestedStruct" is null then null -// else struct_pack( -// "nested_struct_string_field" := "StructField"."NestedStruct"."NestedStructStringField"::varchar -// ) -// end, -// "struct_string_field" := "StructField"."StructStringField"::varchar -// ) -// end as "struct_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "StructField": 'struct("NestedStruct" struct("NestedStructStringField" varchar), "StructStringField" varchar)' -// } -// ))`, -// wantData: []any{"NestedStructStringVal", nil}, -// }, -// { -// name: "nested struct with keyword names", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "end", -// ColumnName: "end", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// { -// SourceName: "any", -// ColumnName: "any", -// Type: "struct", -// StructFields: []*schema.ColumnSchema{ -// { -// SourceName: "for", -// ColumnName: "for", -// Type: "varchar", -// }, -// }, -// }, -// }, -// }, -// }, -// }, -// }, -// json: `{ "end": { "any": { "for": "NestedStructStringVal" }}}`, -// sqlColumn: `"end"."any"."for"`, -// }, -// wantQuery: `select * from (select -// case -// when "end" is null then null -// else struct_pack( -// "any" := case -// when "end"."any" is null then null -// else struct_pack( -// "for" := "end"."any"."for"::varchar -// ) -// end -// ) -// end as "end" -//from -// read_ndjson( -// '%s', -// columns = { -// "end": 'struct("any" struct("for" varchar))' -// } -// ))`, -// wantData: []any{"NestedStructStringVal"}, -// }, -// { -// name: "scalar types", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// {SourceName: "BooleanField", ColumnName: "boolean_field", Type: "boolean"}, -// {SourceName: "TinyIntField", ColumnName: "tinyint_field", Type: "tinyint"}, -// {SourceName: "SmallIntField", ColumnName: "smallint_field", Type: "smallint"}, -// {SourceName: "IntegerField", ColumnName: "integer_field", Type: "integer"}, -// {SourceName: "BigIntField", ColumnName: "bigint_field", Type: "bigint"}, -// {SourceName: "UTinyIntField", ColumnName: "utinyint_field", Type: "utinyint"}, -// {SourceName: "USmallIntField", ColumnName: "usmallint_field", Type: "usmallint"}, -// {SourceName: "UIntegerField", ColumnName: "uinteger_field", Type: "uinteger"}, -// {SourceName: "UBigIntField", ColumnName: "ubigint_field", Type: "ubigint"}, -// {SourceName: "FloatField", ColumnName: "float_field", Type: "float"}, -// {SourceName: "DoubleField", ColumnName: "double_field", Type: "double"}, -// {SourceName: "VarcharField", ColumnName: "varchar_field", Type: "varchar"}, -// {SourceName: "TimestampField", ColumnName: "timestamp_field", Type: "timestamp"}, -// }, -// }, -// }, -// json: `{"BooleanField": true, "TinyIntField": 1, "SmallIntField": 2, "IntegerField": 3, "BigIntField": 4, "UTinyIntField": 5, "USmallIntField": 6, "UIntegerField": 7, "UBigIntField": 8, "FloatField": 1.23, "DoubleField": 4.56, "VarcharField": "StringValue", "TimestampField": "2024-01-01T00:00:00Z"}`, -// sqlColumn: "varchar_field", -// }, -// wantQuery: `select * from (select -// "BooleanField" as "boolean_field", -// "TinyIntField" as "tinyint_field", -// "SmallIntField" as "smallint_field", -// "IntegerField" as "integer_field", -// "BigIntField" as "bigint_field", -// "UTinyIntField" as "utinyint_field", -// "USmallIntField" as "usmallint_field", -// "UIntegerField" as "uinteger_field", -// "UBigIntField" as "ubigint_field", -// "FloatField" as "float_field", -// "DoubleField" as "double_field", -// "VarcharField" as "varchar_field", -// "TimestampField" as "timestamp_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "BooleanField": 'boolean', -// "TinyIntField": 'tinyint', -// "SmallIntField": 'smallint', -// "IntegerField": 'integer', -// "BigIntField": 'bigint', -// "UTinyIntField": 'utinyint', -// "USmallIntField": 'usmallint', -// "UIntegerField": 'uinteger', -// "UBigIntField": 'ubigint', -// "FloatField": 'float', -// "DoubleField": 'double', -// "VarcharField": 'varchar', -// "TimestampField": 'timestamp' -// } -// ))`, -// wantData: []any{"StringValue"}, -// }, -// { -// name: "scalar types - reserved names", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// {SourceName: "end", ColumnName: "end", Type: "boolean"}, -// {SourceName: "for", ColumnName: "for", Type: "tinyint"}, -// }, -// }, -// }, -// json: `{"end": true, "for": 1}`, -// sqlColumn: `"end"`, -// }, -// wantQuery: `select * from (select -// "end" as "end", -// "for" as "for" -//from -// read_ndjson( -// '%s', -// columns = { -// "end": 'boolean', -// "for": 'tinyint' -// } -// ))`, -// wantData: []any{true}, -// }, -// { -// name: "scalar types - missing some data", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// {SourceName: "BooleanField", ColumnName: "boolean_field", Type: "boolean"}, -// {SourceName: "TinyIntField", ColumnName: "tinyint_field", Type: "tinyint"}, -// {SourceName: "SmallIntField", ColumnName: "smallint_field", Type: "smallint"}, -// {SourceName: "IntegerField", ColumnName: "integer_field", Type: "integer"}, -// {SourceName: "BigIntField", ColumnName: "bigint_field", Type: "bigint"}, -// {SourceName: "UTinyIntField", ColumnName: "utinyint_field", Type: "utinyint"}, -// {SourceName: "USmallIntField", ColumnName: "usmallint_field", Type: "usmallint"}, -// {SourceName: "UIntegerField", ColumnName: "uinteger_field", Type: "uinteger"}, -// {SourceName: "UBigIntField", ColumnName: "ubigint_field", Type: "ubigint"}, -// {SourceName: "FloatField", ColumnName: "float_field", Type: "float"}, -// {SourceName: "DoubleField", ColumnName: "double_field", Type: "double"}, -// {SourceName: "VarcharField", ColumnName: "varchar_field", Type: "varchar"}, -// {SourceName: "TimestampField", ColumnName: "timestamp_field", Type: "timestamp"}, -// }, -// }, -// }, -// json: `{"BooleanField": true}`, -// sqlColumn: "boolean_field", -// }, -// wantQuery: `select * from (select -// "BooleanField" as "boolean_field", -// "TinyIntField" as "tinyint_field", -// "SmallIntField" as "smallint_field", -// "IntegerField" as "integer_field", -// "BigIntField" as "bigint_field", -// "UTinyIntField" as "utinyint_field", -// "USmallIntField" as "usmallint_field", -// "UIntegerField" as "uinteger_field", -// "UBigIntField" as "ubigint_field", -// "FloatField" as "float_field", -// "DoubleField" as "double_field", -// "VarcharField" as "varchar_field", -// "TimestampField" as "timestamp_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "BooleanField": 'boolean', -// "TinyIntField": 'tinyint', -// "SmallIntField": 'smallint', -// "IntegerField": 'integer', -// "BigIntField": 'bigint', -// "UTinyIntField": 'utinyint', -// "USmallIntField": 'usmallint', -// "UIntegerField": 'uinteger', -// "UBigIntField": 'ubigint', -// "FloatField": 'float', -// "DoubleField": 'double', -// "VarcharField": 'varchar', -// "TimestampField": 'timestamp' -// } -// ))`, -// wantData: []any{true}, -// }, -// { -// name: "scalar types - some rows missing some data", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// {SourceName: "BooleanField", ColumnName: "boolean_field", Type: "boolean"}, -// {SourceName: "TinyIntField", ColumnName: "tinyint_field", Type: "tinyint"}, -// {SourceName: "SmallIntField", ColumnName: "smallint_field", Type: "smallint"}, -// {SourceName: "IntegerField", ColumnName: "integer_field", Type: "integer"}, -// {SourceName: "BigIntField", ColumnName: "bigint_field", Type: "bigint"}, -// {SourceName: "UTinyIntField", ColumnName: "utinyint_field", Type: "utinyint"}, -// {SourceName: "USmallIntField", ColumnName: "usmallint_field", Type: "usmallint"}, -// {SourceName: "UIntegerField", ColumnName: "uinteger_field", Type: "uinteger"}, -// {SourceName: "UBigIntField", ColumnName: "ubigint_field", Type: "ubigint"}, -// {SourceName: "FloatField", ColumnName: "float_field", Type: "float"}, -// {SourceName: "DoubleField", ColumnName: "double_field", Type: "double"}, -// {SourceName: "VarcharField", ColumnName: "varchar_field", Type: "varchar"}, -// {SourceName: "TimestampField", ColumnName: "timestamp_field", Type: "timestamp"}, -// }, -// }, -// }, -// json: `{"BooleanField": true} -//{"TinyIntField": 1} -//{"TinyIntField": 1, "BooleanField": true}`, -// sqlColumn: "boolean_field", -// }, -// wantQuery: `select * from (select -// "BooleanField" as "boolean_field", -// "TinyIntField" as "tinyint_field", -// "SmallIntField" as "smallint_field", -// "IntegerField" as "integer_field", -// "BigIntField" as "bigint_field", -// "UTinyIntField" as "utinyint_field", -// "USmallIntField" as "usmallint_field", -// "UIntegerField" as "uinteger_field", -// "UBigIntField" as "ubigint_field", -// "FloatField" as "float_field", -// "DoubleField" as "double_field", -// "VarcharField" as "varchar_field", -// "TimestampField" as "timestamp_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "BooleanField": 'boolean', -// "TinyIntField": 'tinyint', -// "SmallIntField": 'smallint', -// "IntegerField": 'integer', -// "BigIntField": 'bigint', -// "UTinyIntField": 'utinyint', -// "USmallIntField": 'usmallint', -// "UIntegerField": 'uinteger', -// "UBigIntField": 'ubigint', -// "FloatField": 'float', -// "DoubleField": 'double', -// "VarcharField": 'varchar', -// "TimestampField": 'timestamp' -// } -// ))`, -// wantData: []any{true, nil, true}, -// }, -// { -// name: "scalar types, missing all data", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// {SourceName: "BooleanField", ColumnName: "boolean_field", Type: "boolean"}, -// {SourceName: "TinyIntField", ColumnName: "tinyint_field", Type: "tinyint"}, -// {SourceName: "SmallIntField", ColumnName: "smallint_field", Type: "smallint"}, -// {SourceName: "IntegerField", ColumnName: "integer_field", Type: "integer"}, -// {SourceName: "BigIntField", ColumnName: "bigint_field", Type: "bigint"}, -// {SourceName: "UTinyIntField", ColumnName: "utinyint_field", Type: "utinyint"}, -// {SourceName: "USmallIntField", ColumnName: "usmallint_field", Type: "usmallint"}, -// {SourceName: "UIntegerField", ColumnName: "uinteger_field", Type: "uinteger"}, -// {SourceName: "UBigIntField", ColumnName: "ubigint_field", Type: "ubigint"}, -// {SourceName: "FloatField", ColumnName: "float_field", Type: "float"}, -// {SourceName: "DoubleField", ColumnName: "double_field", Type: "double"}, -// {SourceName: "VarcharField", ColumnName: "varchar_field", Type: "varchar"}, -// {SourceName: "TimestampField", ColumnName: "timestamp_field", Type: "timestamp"}, -// }, -// }, -// }, -// json: `{}`, -// sqlColumn: "varchar_field", -// }, -// wantQuery: `select * from (select -// "BooleanField" as "boolean_field", -// "TinyIntField" as "tinyint_field", -// "SmallIntField" as "smallint_field", -// "IntegerField" as "integer_field", -// "BigIntField" as "bigint_field", -// "UTinyIntField" as "utinyint_field", -// "USmallIntField" as "usmallint_field", -// "UIntegerField" as "uinteger_field", -// "UBigIntField" as "ubigint_field", -// "FloatField" as "float_field", -// "DoubleField" as "double_field", -// "VarcharField" as "varchar_field", -// "TimestampField" as "timestamp_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "BooleanField": 'boolean', -// "TinyIntField": 'tinyint', -// "SmallIntField": 'smallint', -// "IntegerField": 'integer', -// "BigIntField": 'bigint', -// "UTinyIntField": 'utinyint', -// "USmallIntField": 'usmallint', -// "UIntegerField": 'uinteger', -// "UBigIntField": 'ubigint', -// "FloatField": 'float', -// "DoubleField": 'double', -// "VarcharField": 'varchar', -// "TimestampField": 'timestamp' -// } -// ))`, -// wantData: []any{nil}, -// }, -// { -// name: "array types", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// {SourceName: "BooleanArrayField", ColumnName: "boolean_array_field", Type: "boolean[]"}, -// {SourceName: "TinyIntArrayField", ColumnName: "tinyint_array_field", Type: "tinyint[]"}, -// {SourceName: "SmallIntArrayField", ColumnName: "smallint_array_field", Type: "smallint[]"}, -// {SourceName: "IntegerArrayField", ColumnName: "integer_array_field", Type: "integer[]"}, -// {SourceName: "BigIntArrayField", ColumnName: "bigint_array_field", Type: "bigint[]"}, -// {SourceName: "UTinyIntArrayField", ColumnName: "utinyint_array_field", Type: "utinyint[]"}, -// {SourceName: "USmallIntArrayField", ColumnName: "usmallint_array_field", Type: "usmallint[]"}, -// {SourceName: "UIntegerArrayField", ColumnName: "uinteger_array_field", Type: "uinteger[]"}, -// {SourceName: "UBigIntArrayField", ColumnName: "ubigint_array_field", Type: "ubigint[]"}, -// {SourceName: "FloatArrayField", ColumnName: "float_array_field", Type: "float[]"}, -// {SourceName: "DoubleArrayField", ColumnName: "double_array_field", Type: "double[]"}, -// {SourceName: "VarcharArrayField", ColumnName: "varchar_array_field", Type: "varchar[]"}, -// {SourceName: "TimestampArrayField", ColumnName: "timestamp_array_field", Type: "timestamp[]"}, -// }, -// }, -// }, -// json: `{"BooleanArrayField": [true, false], "TinyIntArrayField": [1, 2], "SmallIntArrayField": [2, 3], "IntegerArrayField": [3, 4], "BigIntArrayField": [4, 5], "UTinyIntArrayField": [5, 6], "USmallIntArrayField": [6, 7], "UIntegerArrayField": [7, 8], "UBigIntArrayField": [8, 9], "FloatArrayField": [1.23, 2.34], "DoubleArrayField": [4.56, 5.67], "VarcharArrayField": ["StringValue1", "StringValue2"], "TimestampArrayField": ["2024-01-01T00:00:00Z", "2024-01-02T00:00:00Z"]}`, -// sqlColumn: "boolean_array_field", -// }, -// wantQuery: `select * from (select -// "BooleanArrayField" as "boolean_array_field", -// "TinyIntArrayField" as "tinyint_array_field", -// "SmallIntArrayField" as "smallint_array_field", -// "IntegerArrayField" as "integer_array_field", -// "BigIntArrayField" as "bigint_array_field", -// "UTinyIntArrayField" as "utinyint_array_field", -// "USmallIntArrayField" as "usmallint_array_field", -// "UIntegerArrayField" as "uinteger_array_field", -// "UBigIntArrayField" as "ubigint_array_field", -// "FloatArrayField" as "float_array_field", -// "DoubleArrayField" as "double_array_field", -// "VarcharArrayField" as "varchar_array_field", -// "TimestampArrayField" as "timestamp_array_field" -//from -// read_ndjson( -// '%s', -// columns = { -// "BooleanArrayField": 'boolean[]', -// "TinyIntArrayField": 'tinyint[]', -// "SmallIntArrayField": 'smallint[]', -// "IntegerArrayField": 'integer[]', -// "BigIntArrayField": 'bigint[]', -// "UTinyIntArrayField": 'utinyint[]', -// "USmallIntArrayField": 'usmallint[]', -// "UIntegerArrayField": 'uinteger[]', -// "UBigIntArrayField": 'ubigint[]', -// "FloatArrayField": 'float[]', -// "DoubleArrayField": 'double[]', -// "VarcharArrayField": 'varchar[]', -// "TimestampArrayField": 'timestamp[]' -// } -// ))`, -// wantData: []any{[]any{true, false}}, -// }, -// { -// name: "array of simple structs", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructArrayField", -// ColumnName: "struct_array_field", -// Type: "struct[]", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "varchar"}, -// {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "integer"}, -// }, -// }, -// }, -// }, -// }, -// json: `{"StructArrayField": [{"StructStringField": "StringValue1", "StructIntField": 1}, {"StructStringField": "StringValue2", "StructIntField": 2}]}`, -// sqlColumn: "struct_array_field[1].struct_string_field", -// }, -// wantQuery: `with raw as ( -// select * from (select -// row_number() over () as rowid, -// "StructArrayField" as "struct_array_field" -// from -// read_ndjson( -// '%s', -// columns = { -// "StructArrayField": 'struct("StructStringField" varchar, "StructIntField" integer)[]' -// } -// )) -//), unnest_struct_array_field as ( -// select -// rowid, -// unnest(coalesce("struct_array_field", array[]::struct("StructStringField" varchar, "StructIntField" integer)[])::struct("StructStringField" varchar, "StructIntField" integer)[]) as struct_array_field -// from -// raw -//), rebuild_unnest_struct_array_field as ( -// select -// rowid, -// struct_array_field->>'StructStringField' as StructArrayField_StructStringField, -// struct_array_field->>'StructIntField' as StructArrayField_StructIntField -// from -// unnest_struct_array_field -//), grouped_unnest_struct_array_field as ( -// select -// rowid, -// array_agg(struct_pack( -// struct_string_field := StructArrayField_StructStringField::varchar, -// struct_int_field := StructArrayField_StructIntField::integer -// )) as struct_array_field -// from -// rebuild_unnest_struct_array_field -// group by -// rowid -//) -//select -// coalesce(joined_struct_array_field.struct_array_field, null) as struct_array_field -//from -// raw -//left join -// grouped_unnest_struct_array_field joined_struct_array_field on raw.rowid = joined_struct_array_field.rowid`, -// wantData: []any{"StringValue1"}, -// }, -// -// // TODO struct arrays are not supported yet -// // in fact one level of struct array field does work, but not nested struct arrays so for -// // now all struct arrays are treated as json -// // { -// // name: "struct with struct array field", -// // args: args{ -// // conversionSchema: &conversionSchema.TableSchema{ -// // Columns: []*conversionSchema.ColumnSchema{ -// // { -// // SourceName: "StructWithArrayField", -// // ColumnName: "struct_with_array_field", -// // Type: "struct", -// // StructFields: []*conversionSchema.ColumnSchema{ -// // {SourceName: "StructArrayField", -// // ColumnName: "struct_array_field", -// // Type: "struct[]", -// // StructFields: []*conversionSchema.ColumnSchema{ -// // {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "VARCHAR"}, -// // {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "INTEGER"}, -// // },}, -// // }, -// // }, -// // }, -// // }, -// // json: `{"StructWithArrayField": {"StructArrayField": [{"StructStringField": "StringValue1", "StructIntField": 1}, {"StructStringField": "StringValue2", "StructIntField": 2}]}}`, -// // sqlColumn: "struct_with_array_field.struct_array_field[1].struct_string_field", -// // }, -// // wantQuery: `WITH raw as ( -// // SELECT -// // row_number() OVER () as rowid, -// // "StructArrayField" as "struct_array_field" -// // FROM -// // read_ndjson( -// // '%s', -// // columns = { -// // "StructArrayField": 'struct("StructStringField" VARCHAR, "StructIntField" INTEGER)[]' -// // } -// // ) -// //), unnest_struct_array_field as ( -// // SELECT -// // rowid, -// // UNNEST(COALESCE("struct_array_field", ARRAY[]::struct("StructStringField" VARCHAR, "StructIntField" INTEGER)[])::struct("StructStringField" VARCHAR, "StructIntField" INTEGER)[]) as struct_array_field -// // FROM -// // raw -// //), rebuild_unnest_struct_array_field as ( -// // SELECT -// // rowid, -// // struct_array_field->>'StructStringField' as StructArrayField_StructStringField, -// // struct_array_field->>'StructIntField' as StructArrayField_StructIntField -// // FROM -// // unnest_struct_array_field -// //), grouped_unnest_struct_array_field as ( -// // SELECT -// // rowid, -// // array_agg(struct_pack( -// // struct_string_field := StructArrayField_StructStringField::VARCHAR, -// // struct_int_field := StructArrayField_StructIntField::INTEGER -// // )) as struct_array_field -// // FROM -// // rebuild_unnest_struct_array_field -// // group by -// // rowid -// //) -// //SELECT -// // COALESCE(joined_struct_array_field.struct_array_field, NULL) as struct_array_field -// //FROM -// // raw -// //left join -// // grouped_unnest_struct_array_field joined_struct_array_field on raw.rowid = joined_struct_array_field.rowid`, -// // wantData: []any{"StringValue1"}, -// // }, -// -// { -// name: "array of simple structs plus other fields", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructArrayField", -// ColumnName: "struct_array_field", -// Type: "struct[]", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "varchar"}, -// {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "integer"}, -// }, -// }, -// {SourceName: "IntField", ColumnName: "int_field", Type: "integer"}, -// {SourceName: "StringField", ColumnName: "string_field", Type: "varchar"}, -// {SourceName: "FloatField", ColumnName: "float_field", Type: "float"}, -// {SourceName: "BooleanField", ColumnName: "boolean_field", Type: "boolean"}, -// { -// SourceName: "IntArrayField", -// ColumnName: "int_array_field", -// Type: "integer[]", -// }, -// { -// SourceName: "StringArrayField", -// ColumnName: "string_array_field", -// Type: "varchar[]", -// }, -// { -// SourceName: "FloatArrayField", -// ColumnName: "float_array_field", -// Type: "float[]", -// }, -// { -// SourceName: "BooleanArrayField", -// ColumnName: "boolean_array_field", -// Type: "boolean[]", -// }, -// }, -// }, -// }, -// -// json: `{"StructArrayField": [{"StructStringField": "StringValue1", "StructIntField": 1}, {"StructStringField": "StringValue2", "StructIntField": 2}], "IntField": 10, "StringField": "SampleString", "FloatField": 10.5, "BooleanField": true, "IntArrayField": [1, 2, 3], "StringArrayField": ["String1", "String2"], "FloatArrayField": [1.1, 2.2, 3.3], "BooleanArrayField": [true, false, true]}`, -// // NOTE: arrays are 1-based -// sqlColumn: "struct_array_field[1].struct_string_field", -// }, -// wantQuery: `with raw as ( -// select * from (select -// row_number() over () as rowid, -// "StructArrayField" as "struct_array_field", -// "IntField" as "int_field", -// "StringField" as "string_field", -// "FloatField" as "float_field", -// "BooleanField" as "boolean_field", -// "IntArrayField" as "int_array_field", -// "StringArrayField" as "string_array_field", -// "FloatArrayField" as "float_array_field", -// "BooleanArrayField" as "boolean_array_field" -// from -// read_ndjson( -// '%s', -// columns = { -// "StructArrayField": 'struct("StructStringField" varchar, "StructIntField" integer)[]', -// "IntField": 'integer', -// "StringField": 'varchar', -// "FloatField": 'float', -// "BooleanField": 'boolean', -// "IntArrayField": 'integer[]', -// "StringArrayField": 'varchar[]', -// "FloatArrayField": 'float[]', -// "BooleanArrayField": 'boolean[]' -// } -// )) -//), unnest_struct_array_field as ( -// select -// rowid, -// unnest(coalesce("struct_array_field", array[]::struct("StructStringField" varchar, "StructIntField" integer)[])::struct("StructStringField" varchar, "StructIntField" integer)[]) as struct_array_field -// from -// raw -//), rebuild_unnest_struct_array_field as ( -// select -// rowid, -// struct_array_field->>'StructStringField' as StructArrayField_StructStringField, -// struct_array_field->>'StructIntField' as StructArrayField_StructIntField -// from -// unnest_struct_array_field -//), grouped_unnest_struct_array_field as ( -// select -// rowid, -// array_agg(struct_pack( -// struct_string_field := StructArrayField_StructStringField::varchar, -// struct_int_field := StructArrayField_StructIntField::integer -// )) as struct_array_field -// from -// rebuild_unnest_struct_array_field -// group by -// rowid -//) -//select -// coalesce(joined_struct_array_field.struct_array_field, null) as struct_array_field, -// raw.int_field, -// raw.string_field, -// raw.float_field, -// raw.boolean_field, -// raw.int_array_field, -// raw.string_array_field, -// raw.float_array_field, -// raw.boolean_array_field -//from -// raw -//left join -// grouped_unnest_struct_array_field joined_struct_array_field on raw.rowid = joined_struct_array_field.rowid`, -// wantData: []any{"StringValue1"}, -// }, -// { -// name: "null array of simple structs plus other fields", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructArrayField", -// ColumnName: "struct_array_field", -// Type: "struct[]", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "varchar"}, -// {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "integer"}, -// }, -// }, -// {SourceName: "IntField", ColumnName: "int_field", Type: "integer"}, -// {SourceName: "StringField", ColumnName: "string_field", Type: "varchar"}, -// {SourceName: "FloatField", ColumnName: "float_field", Type: "float"}, -// {SourceName: "BooleanField", ColumnName: "boolean_field", Type: "boolean"}, -// { -// SourceName: "IntArrayField", -// ColumnName: "int_array_field", -// Type: "integer[]", -// }, -// { -// SourceName: "StringArrayField", -// ColumnName: "string_array_field", -// Type: "varchar[]", -// }, -// { -// SourceName: "FloatArrayField", -// ColumnName: "float_array_field", -// Type: "float[]", -// }, -// { -// SourceName: "BooleanArrayField", -// ColumnName: "boolean_array_field", -// Type: "boolean[]", -// }, -// }, -// }, -// }, -// -// json: `{"StructArrayField": null, "IntField": 10, "StringField": "SampleString", "FloatField": 10.5, "BooleanField": true, "IntArrayField": [1, 2, 3], "StringArrayField": ["String1", "String2"], "FloatArrayField": [1.1, 2.2, 3.3], "BooleanArrayField": [true, false, true]}`, -// sqlColumn: "int_field", -// }, -// wantQuery: `with raw as ( -// select * from (select -// row_number() over () as rowid, -// "StructArrayField" as "struct_array_field", -// "IntField" as "int_field", -// "StringField" as "string_field", -// "FloatField" as "float_field", -// "BooleanField" as "boolean_field", -// "IntArrayField" as "int_array_field", -// "StringArrayField" as "string_array_field", -// "FloatArrayField" as "float_array_field", -// "BooleanArrayField" as "boolean_array_field" -// from -// read_ndjson( -// '%s', -// columns = { -// "StructArrayField": 'struct("StructStringField" varchar, "StructIntField" integer)[]', -// "IntField": 'integer', -// "StringField": 'varchar', -// "FloatField": 'float', -// "BooleanField": 'boolean', -// "IntArrayField": 'integer[]', -// "StringArrayField": 'varchar[]', -// "FloatArrayField": 'float[]', -// "BooleanArrayField": 'boolean[]' -// } -// )) -//), unnest_struct_array_field as ( -// select -// rowid, -// unnest(coalesce("struct_array_field", array[]::struct("StructStringField" varchar, "StructIntField" integer)[])::struct("StructStringField" varchar, "StructIntField" integer)[]) as struct_array_field -// from -// raw -//), rebuild_unnest_struct_array_field as ( -// select -// rowid, -// struct_array_field->>'StructStringField' as StructArrayField_StructStringField, -// struct_array_field->>'StructIntField' as StructArrayField_StructIntField -// from -// unnest_struct_array_field -//), grouped_unnest_struct_array_field as ( -// select -// rowid, -// array_agg(struct_pack( -// struct_string_field := StructArrayField_StructStringField::varchar, -// struct_int_field := StructArrayField_StructIntField::integer -// )) as struct_array_field -// from -// rebuild_unnest_struct_array_field -// group by -// rowid -//) -//select -// coalesce(joined_struct_array_field.struct_array_field, null) as struct_array_field, -// raw.int_field, -// raw.string_field, -// raw.float_field, -// raw.boolean_field, -// raw.int_array_field, -// raw.string_array_field, -// raw.float_array_field, -// raw.boolean_array_field -//from -// raw -//left join -// grouped_unnest_struct_array_field joined_struct_array_field on raw.rowid = joined_struct_array_field.rowid`, -// wantData: []any{int32(10)}, -// }, -// { -// name: "array of simple structs with null value", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructArrayField", -// ColumnName: "struct_array_field", -// Type: "struct[]", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "varchar"}, -// {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "integer"}, -// }, -// }, -// }, -// }, -// }, -// json: `{"StructArrayField": null}`, -// sqlColumn: "struct_array_field", -// }, -// wantQuery: `with raw as ( -// select * from (select -// row_number() over () as rowid, -// "StructArrayField" as "struct_array_field" -// from -// read_ndjson( -// '%s', -// columns = { -// "StructArrayField": 'struct("StructStringField" varchar, "StructIntField" integer)[]' -// } -// )) -//), unnest_struct_array_field as ( -// select -// rowid, -// unnest(coalesce("struct_array_field", array[]::struct("StructStringField" varchar, "StructIntField" integer)[])::struct("StructStringField" varchar, "StructIntField" integer)[]) as struct_array_field -// from -// raw -//), rebuild_unnest_struct_array_field as ( -// select -// rowid, -// struct_array_field->>'StructStringField' as StructArrayField_StructStringField, -// struct_array_field->>'StructIntField' as StructArrayField_StructIntField -// from -// unnest_struct_array_field -//), grouped_unnest_struct_array_field as ( -// select -// rowid, -// array_agg(struct_pack( -// struct_string_field := StructArrayField_StructStringField::varchar, -// struct_int_field := StructArrayField_StructIntField::integer -// )) as struct_array_field -// from -// rebuild_unnest_struct_array_field -// group by -// rowid -//) -//select -// coalesce(joined_struct_array_field.struct_array_field, null) as struct_array_field -//from -// raw -//left join -// grouped_unnest_struct_array_field joined_struct_array_field on raw.rowid = joined_struct_array_field.rowid`, -// wantData: []any{nil}, -// }, -// { -// name: "array of simple structs with null value and non null value", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructArrayField", -// ColumnName: "struct_array_field", -// Type: "struct[]", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "varchar"}, -// {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "integer"}, -// }, -// }, -// }, -// }, -// }, -// json: `{"StructArrayField": null} -//{"StructArrayField": [{"StructStringField": "StringValue1", "StructIntField": 1}, {"StructStringField": "StringValue2", "StructIntField": 2}]}`, -// sqlColumn: "struct_array_field[1].struct_string_field", -// }, -// wantQuery: `with raw as ( -// select * from (select -// row_number() over () as rowid, -// "StructArrayField" as "struct_array_field" -// from -// read_ndjson( -// '%s', -// columns = { -// "StructArrayField": 'struct("StructStringField" varchar, "StructIntField" integer)[]' -// } -// )) -//), unnest_struct_array_field as ( -// select -// rowid, -// unnest(coalesce("struct_array_field", array[]::struct("StructStringField" varchar, "StructIntField" integer)[])::struct("StructStringField" varchar, "StructIntField" integer)[]) as struct_array_field -// from -// raw -//), rebuild_unnest_struct_array_field as ( -// select -// rowid, -// struct_array_field->>'StructStringField' as StructArrayField_StructStringField, -// struct_array_field->>'StructIntField' as StructArrayField_StructIntField -// from -// unnest_struct_array_field -//), grouped_unnest_struct_array_field as ( -// select -// rowid, -// array_agg(struct_pack( -// struct_string_field := StructArrayField_StructStringField::varchar, -// struct_int_field := StructArrayField_StructIntField::integer -// )) as struct_array_field -// from -// rebuild_unnest_struct_array_field -// group by -// rowid -//) -//select -// coalesce(joined_struct_array_field.struct_array_field, null) as struct_array_field -//from -// raw -//left join -// grouped_unnest_struct_array_field joined_struct_array_field on raw.rowid = joined_struct_array_field.rowid`, -// //wantData: []any{nil, "StringValue1"}, -// // NOTE: ordering is not guaranteed -// wantData: []any{"StringValue1", nil}, -// }, -// { -// name: "2 arrays of simple structs", -// args: args{ -// schema: &schema.ConversionSchema{ -// TableSchema: schema.TableSchema{ -// Columns: []*schema.ColumnSchema{ -// { -// SourceName: "StructArrayField", -// ColumnName: "struct_array_field", -// Type: "struct[]", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField", ColumnName: "struct_string_field", Type: "varchar"}, -// {SourceName: "StructIntField", ColumnName: "struct_int_field", Type: "integer"}, -// }, -// }, -// { -// SourceName: "StructArrayField2", -// ColumnName: "struct_array_field2", -// Type: "struct[]", -// StructFields: []*schema.ColumnSchema{ -// {SourceName: "StructStringField2", ColumnName: "struct_string_field2", Type: "varchar"}, -// {SourceName: "StructIntField2", ColumnName: "struct_int_field2", Type: "integer"}, -// }, -// }, -// }, -// }, -// }, -// json: `{"StructArrayField": [{"StructStringField": "StringValue1", "StructIntField": 1}, {"StructStringField": "StringValue2", "StructIntField": 2}], "StructArrayField2": [{"StructStringField2": "StringValue100", "StructIntField2": 100}, {"StructStringField2": "StringValue200", "StructIntField2": 200}]}`, -// sqlColumn: "struct_array_field2[1].struct_string_field2", -// }, -// wantQuery: `with raw as ( -// select * from (select -// row_number() over () as rowid, -// "StructArrayField" as "struct_array_field", -// "StructArrayField2" as "struct_array_field2" -// from -// read_ndjson( -// '%s', -// columns = { -// "StructArrayField": 'struct("StructStringField" varchar, "StructIntField" integer)[]', -// "StructArrayField2": 'struct("StructStringField2" varchar, "StructIntField2" integer)[]' -// } -// )) -//), unnest_struct_array_field as ( -// select -// rowid, -// unnest(coalesce("struct_array_field", array[]::struct("StructStringField" varchar, "StructIntField" integer)[])::struct("StructStringField" varchar, "StructIntField" integer)[]) as struct_array_field -// from -// raw -//), rebuild_unnest_struct_array_field as ( -// select -// rowid, -// struct_array_field->>'StructStringField' as StructArrayField_StructStringField, -// struct_array_field->>'StructIntField' as StructArrayField_StructIntField -// from -// unnest_struct_array_field -//), grouped_unnest_struct_array_field as ( -// select -// rowid, -// array_agg(struct_pack( -// struct_string_field := StructArrayField_StructStringField::varchar, -// struct_int_field := StructArrayField_StructIntField::integer -// )) as struct_array_field -// from -// rebuild_unnest_struct_array_field -// group by -// rowid -//), unnest_struct_array_field2 as ( -// select -// rowid, -// unnest(coalesce("struct_array_field2", array[]::struct("StructStringField2" varchar, "StructIntField2" integer)[])::struct("StructStringField2" varchar, "StructIntField2" integer)[]) as struct_array_field2 -// from -// raw -//), rebuild_unnest_struct_array_field2 as ( -// select -// rowid, -// struct_array_field2->>'StructStringField2' as StructArrayField2_StructStringField2, -// struct_array_field2->>'StructIntField2' as StructArrayField2_StructIntField2 -// from -// unnest_struct_array_field2 -//), grouped_unnest_struct_array_field2 as ( -// select -// rowid, -// array_agg(struct_pack( -// struct_string_field2 := StructArrayField2_StructStringField2::varchar, -// struct_int_field2 := StructArrayField2_StructIntField2::integer -// )) as struct_array_field2 -// from -// rebuild_unnest_struct_array_field2 -// group by -// rowid -//) -//select -// coalesce(joined_struct_array_field.struct_array_field, null) as struct_array_field, -// coalesce(joined_struct_array_field2.struct_array_field2, null) as struct_array_field2 -//from -// raw -//left join -// grouped_unnest_struct_array_field joined_struct_array_field on raw.rowid = joined_struct_array_field.rowid -//left join -// grouped_unnest_struct_array_field2 joined_struct_array_field2 on raw.rowid = joined_struct_array_field2.rowid`, -// wantData: []any{"StringValue100"}, -// }, -// // TODO #parquet https://github.com/turbot/tailpipe/issues/new -// // { -// // name: "map types", -// // args: args{ -// // conversionSchema: &conversionSchema.TableSchema{ -// // Columns: []*conversionSchema.ColumnSchema{ -// // {SourceName: "BooleanMapField", ColumnName: "boolean_map_field", Type: "map"}, -// // {SourceName: "TinyIntMapField", ColumnName: "tinyint_map_field", Type: "map"}, -// // {SourceName: "SmallIntMapField", ColumnName: "smallint_map_field", Type: "map"}, -// // {SourceName: "IntegerMapField", ColumnName: "integer_map_field", Type: "map"}, -// // {SourceName: "BigIntMapField", ColumnName: "bigint_map_field", Type: "map"}, -// // {SourceName: "FloatMapField", ColumnName: "float_map_field", Type: "map"}, -// // {SourceName: "DoubleMapField", ColumnName: "double_map_field", Type: "map"}, -// // {SourceName: "VarcharMapField", ColumnName: "varchar_map_field", Type: "map"}, -// // {SourceName: "TimestampMapField", ColumnName: "timestamp_map_field", Type: "map"}, -// // }, -// // }, -// // json: `{"BooleanMapField": {"key1": true, "key2": false}, "TinyIntMapField": {"key1": 1, "key2": 2}, "SmallIntMapField": {"key1": 2, "key2": 3}, "IntegerMapField": {"key1": 3, "key2": 4}, "BigIntMapField": {"key1": 4, "key2": 5}, "FloatMapField": {"key1": 1.23, "key2": 2.34}, "DoubleMapField": {"key1": 4.56, "key2": 5.67}, "VarcharMapField": {"key1": "StringValue1", "key2": "StringValue2"}, "TimestampMapField": {"key1": "2024-01-01T00:00:00Z", "key2": "2024-01-02T00:00:00Z"}}`, -// // sqlColumn: "boolean_map_field", -// // }, -// // wantQuery: `select -// // json_extract(json, '$.BooleanMapField')::map(varchar, boolean> as boolean_map_field, -// // json_extract(json, '$.TinyIntMapField')::map(varchar, tinyint> as tinyint_map_field, -// // json_extract(json, '$.SmallIntMapField')::map(varchar, smallint) as smallint_map_field, -// // json_extract(json, '$.IntegerMapField')::map(varchar, integer) as integer_map_field, -// // json_extract(json, '$.BigIntMapField')::map(varchar, bigint) as bigint_map_field, -// // json_extract(json, '$.FloatMapField')::map(varchar, float) as float_map_field, -// // json_extract(json, '$.DoubleMapField')::map(varchar, double) as double_map_field, -// // json_extract(json, '$.VarcharMapField')::map(varchar, varchar) as varchar_map_field, -// // json_extract(json, '$.TimestampMapField')::map(varchar, timestamp) as timestamp_map_field -// //from read_json_auto('%s', format='newline_delimited')`, jsonlFilePath), -// // wantData: map[string]bool{"key1": true, "key2": false}, -// // }, -// } -// -// defer os.RemoveAll("test_data") -// -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// conversionSchema := schema.NewConversionSchema(&tt.args.schema.TableSchema) -// query := buildViewQuery(conversionSchema) -// -// // first check the quey is as expected -// if query != tt.wantQuery { -// t.Errorf("buildViewQuery(), got:\n%s\nwant:\n%s", query, tt.wantQuery) -// } -// -// gotData, err := executeQuery(t, query, tt.args.json, tt.args.sqlColumn) -// if err != nil { -// t.Errorf("error executing query: %s", err) -// } else if !reflect.DeepEqual(gotData, tt.wantData) { -// t.Errorf("buildViewQuery() query returned %v, want %v", gotData, tt.wantData) -// } -// }) -// } -//} -// -//func executeQuery(t *testing.T, queryFormat, json, sqlColumn string) (any, error) { -// -// // now verify the query runs -// // copy json to a jsonl file -// err := createJSONLFile(json) -// if err != nil { -// t.Fatalf("error creating jsonl file: %s", err) -// } -// defer os.Remove(jsonlFilePath) -// -// // render query with the file path -// query := fmt.Sprintf(queryFormat, jsonlFilePath) -// -// // get the data -// var data []any -// -// // execute in duckdb -// // build select queryz -// testQuery := fmt.Sprintf("select %s from (%s)", sqlColumn, query) -// rows, err := testDb.Query(testQuery) -// -// if err != nil { -// return nil, fmt.Errorf("error executing query: %w", err) -// } -// // Iterate over the results -// for rows.Next() { -// var d any -// -// if err := rows.Scan(&d); err != nil { -// return nil, fmt.Errorf("error scanning data: %w", err) -// } -// data = append(data, d) -// } -// -// return data, nil -//} -// -//func createJSONLFile(json string) error { -// // remove just in case -// os.Remove(jsonlFilePath) -// jsonlFile, err := os.Create(jsonlFilePath) -// if err != nil { -// return fmt.Errorf("error creating jsonl file: %w", err) -// } -// _, err = jsonlFile.WriteString(json) -// if err != nil { -// return fmt.Errorf("error writing to jsonl file: %w", err) -// } -// // close the file -// err = jsonlFile.Close() -// if err != nil { -// return fmt.Errorf("error closing jsonl file: %w", err) -// } -// return err -//} -// -// TODO KAI re-add -// -//func TestBuildValidationQuery(t *testing.T) { -// testCases := []struct { -// name string -// selectQuery string -// columnsToValidate []string -// expectedQuery string -// }{ -// { -// name: "single column", -// selectQuery: "select * from source", -// columnsToValidate: []string{"name"}, -// expectedQuery: `drop table if exists temp_data; -//create temp table temp_data as select * from source; -//select -// count(*) as total_rows, -// list(distinct col) as columns_with_nulls -//from ( -// select 'name' as col from temp_data where name is null -//) -//`, -// }, -// { -// name: "multiple columns", -// selectQuery: "select * from source", -// columnsToValidate: []string{"name", "email", "age"}, -// expectedQuery: `drop table if exists temp_data; -//create temp table temp_data as select * from source; -//select -// count(*) as total_rows, -// list(distinct col) as columns_with_nulls -//from ( -// select 'name' as col from temp_data where name is null -// union all -// select 'email' as col from temp_data where email is null -// union all -// select 'age' as col from temp_data where age is null -//) -//`, -// }, -// { -// name: "no columns", -// selectQuery: "select * from source", -// columnsToValidate: []string{}, -// expectedQuery: `drop table if exists temp_data; -//create temp table temp_data as select * from source; -//select -// count(*) as total_rows, -// list(distinct col) as columns_with_nulls -//from ( -//) -//`, -// }, -// } -// -// for _, tc := range testCases { -// t.Run(tc.name, func(t *testing.T) { -// worker := &conversionWorker{} -// actualQuery := worker.buildValidationQuery(tc.columnsToValidate) -// assert.Equal(t, tc.expectedQuery, actualQuery) -// }) -// } -//} diff --git a/internal/parquet/convertor.go b/internal/parquet/convertor.go deleted file mode 100644 index 9c4dc78f..00000000 --- a/internal/parquet/convertor.go +++ /dev/null @@ -1,245 +0,0 @@ -package parquet - -import ( - "context" - "errors" - "fmt" - "log/slog" - "sync" - "sync/atomic" - - "github.com/turbot/tailpipe-plugin-sdk/schema" - "github.com/turbot/tailpipe/internal/config" -) - -const parquetWorkerCount = 5 -const chunkBufferLength = 1000 - -// Converter struct executes all the conversions for a single collection -// it therefore has a unique execution id, and will potentially convert of multiple JSONL files -// each file is assumed to have the filename format _.jsonl -// so when new input files are available, we simply store the chunk number -type Converter struct { - // the execution id - id string - - // the file chunks numbers available to process - chunks []int32 - chunkLock sync.Mutex - chunkSignal *sync.Cond - // the channel to send execution to the workers - jobChan chan *parquetJob - // waitGroup to track job completion - wg sync.WaitGroup - // the cancel function for the context used to manage the job - cancel context.CancelFunc - - // the number of chunks processed so far - completionCount int32 - // the number of rows written - rowCount int64 - // the number of rows which were NOT converted due to conversion errors encountered - failedRowCount int64 - - // the source file location - sourceDir string - // the dest file location - destDir string - // helper to provide unique file roots - fileRootProvider *FileRootProvider - - // the format string for the conversion query will be the same for all chunks - build once and store - viewQueryFormat string - - // the table conversionSchema - populated when the first chunk arrives if the conversionSchema is not already complete - conversionSchema *schema.ConversionSchema - // the source schema - used to build the conversionSchema - tableSchema *schema.TableSchema - - // viewQueryOnce ensures the schema inference only happens once for the first chunk, - // even if multiple chunks arrive concurrently. Combined with schemaWg, this ensures - // all subsequent chunks wait for the initial schema inference to complete before proceeding. - viewQueryOnce sync.Once - // schemaWg is used to block processing of subsequent chunks until the initial - // schema inference is complete. This ensures all chunks wait for the schema - // to be fully initialized before proceeding with their processing. - schemaWg sync.WaitGroup - - // the partition being collected - Partition *config.Partition - // func which we call with updated row count - statusFunc func(int64, int64, ...error) -} - -func NewParquetConverter(ctx context.Context, cancel context.CancelFunc, executionId string, partition *config.Partition, sourceDir string, tableSchema *schema.TableSchema, statusFunc func(int64, int64, ...error)) (*Converter, error) { - // get the data dir - this will already have been created by the config loader - destDir := config.GlobalWorkspaceProfile.GetDataDir() - - // normalise the table schema to use lowercase column names - tableSchema.NormaliseColumnTypes() - - w := &Converter{ - id: executionId, - chunks: make([]int32, 0, chunkBufferLength), // Pre-allocate reasonable capacity - Partition: partition, - jobChan: make(chan *parquetJob, parquetWorkerCount*2), - cancel: cancel, - sourceDir: sourceDir, - destDir: destDir, - tableSchema: tableSchema, - statusFunc: statusFunc, - fileRootProvider: &FileRootProvider{}, - } - // create the condition variable using the same lock - w.chunkSignal = sync.NewCond(&w.chunkLock) - - // start the goroutine to schedule the jobs - go w.scheduler(ctx) - - // start the workers - for range parquetWorkerCount { - wk, err := newParquetConversionWorker(w) - if err != nil { - return nil, fmt.Errorf("failed to create worker: %w", err) - } - // start the worker - go wk.start(ctx) - } - - // done - return w, nil -} - -func (w *Converter) Close() { - slog.Info("closing Converter") - // close the close channel to signal to the job schedulers to exit - w.cancel() -} - -// AddChunk adds a new chunk to the list of chunks to be processed -// if this is the first chunk, determine if we have a full conversionSchema yet and if not infer from the chunk -// signal the scheduler that `chunks are available -func (w *Converter) AddChunk(executionId string, chunk int32) error { - var err error - w.schemaWg.Wait() - - // Execute schema inference exactly once for the first chunk. - // The WaitGroup ensures all subsequent chunks wait for this to complete. - // If schema inference fails, the error is captured and returned to the caller. - w.viewQueryOnce.Do(func() { - w.schemaWg.Add(1) - defer w.schemaWg.Done() - if err = w.buildConversionSchema(executionId, chunk); err != nil { - // err will be returned by the parent function - return - } - w.viewQueryFormat = buildViewQuery(w.conversionSchema) - }) - if err != nil { - return fmt.Errorf("failed to infer schema: %w", err) - } - w.chunkLock.Lock() - w.chunks = append(w.chunks, chunk) - w.chunkLock.Unlock() - - w.wg.Add(1) - - // Signal that new chunk is available - // Using Signal instead of Broadcast as only one worker needs to wake up - w.chunkSignal.Signal() - - return nil -} - -// WaitForConversions waits for all jobs to be processed or for the context to be cancelled -func (w *Converter) WaitForConversions(ctx context.Context) { - slog.Info("Converter.WaitForConversions - waiting for all jobs to be processed or context to be cancelled.") - // wait for the wait group within a goroutine so we can also check the context - done := make(chan struct{}) - go func() { - w.wg.Wait() - close(done) - }() - - select { - case <-ctx.Done(): - slog.Info("WaitForConversions - context cancelled.") - case <-done: - slog.Info("WaitForConversions - all jobs processed.") - } -} - -// waitForSignal waits for the condition signal or context cancellation -// returns true if context was cancelled -func (w *Converter) waitForSignal(ctx context.Context) bool { - w.chunkLock.Lock() - defer w.chunkLock.Unlock() - - select { - case <-ctx.Done(): - return true - default: - w.chunkSignal.Wait() - return false - } -} - -// the scheduler is responsible for sending jobs to the workere -// it listens for signals on the chunkWrittenSignal channel and enqueues jobs when they arrive -func (w *Converter) scheduler(ctx context.Context) { - defer close(w.jobChan) - - for { - chunk, ok := w.getNextChunk() - if !ok { - if w.waitForSignal(ctx) { - slog.Debug("scheduler shutting down due to context cancellation") - return - } - continue - } - - select { - case <-ctx.Done(): - return - case w.jobChan <- &parquetJob{chunkNumber: chunk}: - } - } -} - -func (w *Converter) getNextChunk() (int32, bool) { - w.chunkLock.Lock() - defer w.chunkLock.Unlock() - - if len(w.chunks) == 0 { - return 0, false - } - - // Take from end - more efficient as it avoids shifting elements - lastIdx := len(w.chunks) - 1 - chunk := w.chunks[lastIdx] - w.chunks = w.chunks[:lastIdx] - return chunk, true -} - -func (w *Converter) addJobErrors(errorList ...error) { - var failedRowCount int64 - - for _, err := range errorList { - var conversionError = &ConversionError{} - if errors.As(err, &conversionError) { - failedRowCount = atomic.AddInt64(&w.failedRowCount, conversionError.RowsAffected) - } - slog.Error("conversion error", "error", err) - } - - // update the status function with the new error count (no need to use atomic for errorList as we are already locked) - w.statusFunc(atomic.LoadInt64(&w.rowCount), failedRowCount, errorList...) -} - -// updateRowCount atomically increments the row count and calls the statusFunc -func (w *Converter) updateRowCount(count int64) { - atomic.AddInt64(&w.rowCount, count) - // call the status function with the new row count - w.statusFunc(atomic.LoadInt64(&w.rowCount), atomic.LoadInt64(&w.failedRowCount)) -} diff --git a/internal/parquet/delete.go b/internal/parquet/delete.go deleted file mode 100644 index 54b6dc74..00000000 --- a/internal/parquet/delete.go +++ /dev/null @@ -1,163 +0,0 @@ -package parquet - -import ( - "fmt" - "log/slog" - "os" - "path/filepath" - "strings" - "time" - - "github.com/turbot/pipe-fittings/v2/utils" - "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/database" - "github.com/turbot/tailpipe/internal/filepaths" -) - -func DeleteParquetFiles(partition *config.Partition, from time.Time) (rowCount int, err error) { - db, err := database.NewDuckDb() - if err != nil { - return 0, fmt.Errorf("failed to open DuckDB connection: %w", err) - } - defer db.Close() - - dataDir := config.GlobalWorkspaceProfile.GetDataDir() - - if from.IsZero() { - // if there is no from time, delete the entire partition folder - rowCount, err = deletePartition(db, dataDir, partition) - } else { - // otherwise delete partition data for a time range - rowCount, err = deletePartitionFrom(db, dataDir, partition, from) - } - if err != nil { - return 0, fmt.Errorf("failed to delete partition: %w", err) - } - - // delete all empty folders underneath the partition folder - partitionDir := filepaths.GetParquetPartitionPath(dataDir, partition.TableName, partition.ShortName) - pruneErr := filepaths.PruneTree(partitionDir) - if pruneErr != nil { - // do not return error - just log - slog.Warn("DeleteParquetFiles failed to prune empty folders", "error", pruneErr) - } - - return rowCount, nil -} - -func deletePartitionFrom(db *database.DuckDb, dataDir string, partition *config.Partition, from time.Time) (_ int, err error) { - parquetGlobPath := filepaths.GetParquetFileGlobForPartition(dataDir, partition.TableName, partition.ShortName, "") - - query := fmt.Sprintf(` - select - distinct '%s/tp_table=' || tp_table || '/tp_partition=' || tp_partition || '/tp_index=' || tp_index || '/tp_date=' || tp_date as hive_path, - count(*) over() as total_files - from read_parquet('%s', hive_partitioning=true) - where tp_partition = ? - and tp_date >= ?`, - dataDir, parquetGlobPath) - - rows, err := db.Query(query, partition.ShortName, from) - if err != nil { - // is this an error because there are no files? - if isNoFilesFoundError(err) { - return 0, nil - } - return 0, fmt.Errorf("failed to query parquet folder names: %w", err) - } - defer rows.Close() - - var folders []string - var count int - // Iterate over the results - for rows.Next() { - var folder string - if err := rows.Scan(&folder, &count); err != nil { - return 0, fmt.Errorf("failed to scan parquet folder name: %w", err) - } - folders = append(folders, folder) - } - - var errors = make(map[string]error) - for _, folder := range folders { - if err := os.RemoveAll(folder); err != nil { - errors[folder] = err - } - } - - return len(folders), nil -} - -func deletePartition(db *database.DuckDb, dataDir string, partition *config.Partition) (int, error) { - parquetGlobPath := filepaths.GetParquetFileGlobForPartition(dataDir, partition.TableName, partition.ShortName, "") - - // get count of parquet files - query := fmt.Sprintf(` - select count(distinct filename) - from read_parquet('%s', hive_partitioning=true, filename=true) - where tp_partition = ? - `, parquetGlobPath) - - // Execute the query with a parameter for the tp_partition filter - q := db.QueryRow(query, partition.ShortName) - // read the result - var count int - err := q.Scan(&count) - if err != nil && !isNoFilesFoundError(err) { - return 0, fmt.Errorf("failed to query parquet file count: %w", err) - } - - partitionFolder := filepaths.GetParquetPartitionPath(dataDir, partition.TableName, partition.ShortName) - err = os.RemoveAll(partitionFolder) - if err != nil { - return 0, fmt.Errorf("failed to delete partition folder: %w", err) - } - return count, nil -} - -func isNoFilesFoundError(err error) bool { - return strings.HasPrefix(err.Error(), "IO Error: No files found") -} - -//// getDeleteInvalidDate determines the date from which to delete invalid files -//// It returns the later of the from date and the InvalidFromDate -//func getDeleteInvalidDate(from, invalidFromDate time.Time) time.Time { -// deleteInvalidDate := from -// if invalidFromDate.After(from) { -// deleteInvalidDate = invalidFromDate -// } -// return deleteInvalidDate -//} - -// deleteInvalidParquetFiles deletes invalid and temporary parquet files for a partition -func deleteInvalidParquetFiles(dataDir string, patterns []PartitionPattern) error { - var failures int - - for _, pattern := range patterns { - - slog.Info("deleteInvalidParquetFiles - deleting invalid parquet files", "table", pattern.Table, "partition", pattern.Partition) - - // get glob patterns for invalid and temp files - invalidGlob := filepaths.GetTempAndInvalidParquetFileGlobForPartition(dataDir, pattern.Table, pattern.Partition) - - // find all matching files - filesToDelete, err := filepath.Glob(invalidGlob) - if err != nil { - return fmt.Errorf("failed to find invalid files: %w", err) - } - - slog.Info("deleteInvalidParquetFiles", "invalid count", len(filesToDelete), "files", filesToDelete) - - // delete each file - for _, file := range filesToDelete { - if err := os.Remove(file); err != nil { - slog.Debug("failed to delete invalid parquet file", "file", file, "error", err) - failures++ - } - } - } - if failures > 0 { - return fmt.Errorf("failed to delete %d invalid parquet %s", failures, utils.Pluralize("file", failures)) - } - return nil -} diff --git a/internal/parquet/delete_test.go b/internal/parquet/delete_test.go deleted file mode 100644 index aeddb9b7..00000000 --- a/internal/parquet/delete_test.go +++ /dev/null @@ -1,356 +0,0 @@ -package parquet - -import ( - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/hashicorp/hcl/v2" - "github.com/stretchr/testify/assert" - "github.com/turbot/tailpipe/internal/config" - "github.com/turbot/tailpipe/internal/filepaths" -) - -func Test_deleteInvalidParquetFiles(t *testing.T) { - // Create a temporary directory for test files - tempDir, err := os.MkdirTemp("", "delete_invalid_parquet_test") - if err != nil { - t.Fatalf("Failed to create temp dir: %v", err) - } - defer os.RemoveAll(tempDir) - - // Create test partition - block := &hcl.Block{ - Labels: []string{"test_table", "test_partition"}, - } - partitionResource, _ := config.NewPartition(block, "partition.test_table.test_partition") - partition := partitionResource.(*config.Partition) - - // Create test files - testFiles := []struct { - name string - expected bool // whether the file should be deleted - }{ - { - name: "old_invalid.parquet.invalid", - expected: true, - }, - { - name: "new_invalid.parquet.invalid", - expected: true, - }, - { - name: "old_temp.parquet.tmp", - expected: true, - }, - { - name: "new_temp.parquet.tmp", - expected: true, - }, - { - name: "valid.parquet", - expected: false, - }, - } - - // Create the partition directory - partitionDir := filepaths.GetParquetPartitionPath(tempDir, partition.TableName, partition.ShortName) - if err := os.MkdirAll(partitionDir, 0755); err != nil { - t.Fatalf("Failed to create partition dir: %v", err) - } - - // Create test files - for _, tf := range testFiles { - filePath := filepath.Join(partitionDir, tf.name) - if err := os.WriteFile(filePath, []byte("test data"), 0644); err != nil { //nolint:gosec // test code - t.Fatalf("Failed to create test file %s: %v", tf.name, err) - } - } - - // Debug: Print directory structure - err = filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - rel, _ := filepath.Rel(tempDir, path) - if rel == "." { - return nil - } - if info.IsDir() { - t.Logf("DIR: %s", rel) - } else { - t.Logf("FILE: %s", rel) - } - return nil - }) - if err != nil { - t.Logf("Error walking directory: %v", err) - } - - // Debug: Print glob pattern - invalidGlob := filepaths.GetTempAndInvalidParquetFileGlobForPartition(tempDir, partition.TableName, partition.ShortName) - t.Logf("Glob pattern: %s", invalidGlob) - - // Run the delete function - patterns := []PartitionPattern{NewPartitionPattern(partition)} - err = deleteInvalidParquetFiles(tempDir, patterns) - if err != nil { - t.Fatalf("deleteInvalidParquetFiles failed: %v", err) - } - - // Check which files were deleted - for _, tf := range testFiles { - filePath := filepath.Join(partitionDir, tf.name) - _, err := os.Stat(filePath) - exists := err == nil - - if tf.expected { - assert.False(t, exists, "File %s should have been deleted", tf.name) - } else { - assert.True(t, exists, "File %s should not have been deleted", tf.name) - } - } -} - -func Test_deleteInvalidParquetFilesWithWildcards(t *testing.T) { - // Create a temporary directory for test files - tempDir, err := os.MkdirTemp("", "delete_invalid_parquet_test") - if err != nil { - t.Fatalf("Failed to create temp dir: %v", err) - } - defer os.RemoveAll(tempDir) - - // Create test partitions - partitions := []struct { - table string - partition string - }{ - {"aws_cloudtrail", "cloudtrail"}, - {"aws_cloudtrail", "cloudwatch"}, - {"aws_ec2", "instances"}, - {"aws_ec2", "volumes"}, - } - - // Create test files for each partition - testFiles := []struct { - name string - expected bool - }{ - { - name: "invalid.parquet.invalid", - expected: true, - }, - { - name: "temp.parquet.tmp", - expected: true, - }, - { - name: "valid.parquet", - expected: false, - }, - } - - // Create directories and files for each partition - for _, p := range partitions { - partitionDir := filepaths.GetParquetPartitionPath(tempDir, p.table, p.partition) - if err := os.MkdirAll(partitionDir, 0755); err != nil { - t.Fatalf("Failed to create partition dir: %v", err) - } - - for _, tf := range testFiles { - filePath := filepath.Join(partitionDir, tf.name) - if err := os.WriteFile(filePath, []byte("test data"), 0644); err != nil { //nolint:gosec // test code - t.Fatalf("Failed to create test file %s: %v", tf.name, err) - } - } - } - - // Test cases with different wildcard patterns - tests := []struct { - name string - patterns []PartitionPattern - deleted map[string]bool // key is "table/partition", value is whether files should be deleted - }{ - { - name: "match all aws_cloudtrail partitions", - patterns: []PartitionPattern{{ - Table: "aws_cloudtrail", - Partition: "*", - }}, - deleted: map[string]bool{ - "aws_cloudtrail/cloudtrail": true, - "aws_cloudtrail/cloudwatch": true, - "aws_ec2/instances": false, - "aws_ec2/volumes": false, - }, - }, - { - name: "match all aws_* tables", - patterns: []PartitionPattern{{ - Table: "aws_*", - Partition: "*", - }}, - deleted: map[string]bool{ - "aws_cloudtrail/cloudtrail": true, - "aws_cloudtrail/cloudwatch": true, - "aws_ec2/instances": true, - "aws_ec2/volumes": true, - }, - }, - { - name: "match specific partitions across tables", - patterns: []PartitionPattern{ - {Table: "aws_cloudtrail", Partition: "cloudtrail"}, - {Table: "aws_ec2", Partition: "instances"}, - }, - deleted: map[string]bool{ - "aws_cloudtrail/cloudtrail": true, - "aws_cloudtrail/cloudwatch": false, - "aws_ec2/instances": true, - "aws_ec2/volumes": false, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Run the delete function - err = deleteInvalidParquetFiles(tempDir, tt.patterns) - if err != nil { - t.Fatalf("deleteInvalidParquetFiles failed: %v", err) - } - - // Check each partition - for _, p := range partitions { - partitionDir := filepaths.GetParquetPartitionPath(tempDir, p.table, p.partition) - key := fmt.Sprintf("%s/%s", p.table, p.partition) - shouldDelete := tt.deleted[key] - - // Check each file - for _, tf := range testFiles { - filePath := filepath.Join(partitionDir, tf.name) - _, err := os.Stat(filePath) - exists := err == nil - - if shouldDelete && tf.expected { - assert.False(t, exists, "[%s] File %s should have been deleted", key, tf.name) - } else { - assert.True(t, exists, "[%s] File %s should not have been deleted", key, tf.name) - } - } - } - - // Recreate the files for the next test - for _, p := range partitions { - partitionDir := filepaths.GetParquetPartitionPath(tempDir, p.table, p.partition) - for _, tf := range testFiles { - filePath := filepath.Join(partitionDir, tf.name) - if err := os.WriteFile(filePath, []byte("test data"), 0644); err != nil { //nolint:gosec // test code - t.Fatalf("Failed to recreate test file %s: %v", tf.name, err) - } - } - } - }) - } -} - -//func Test_shouldClearInvalidState(t *testing.T) { -// tests := []struct { -// name string -// invalidFromDate time.Time -// from time.Time -// want bool -// }{ -// { -// name: "both zero", -// invalidFromDate: time.Time{}, -// from: time.Time{}, -// want: true, -// }, -// { -// name: "invalidFromDate zero, from not zero", -// invalidFromDate: time.Time{}, -// from: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// want: false, -// }, -// { -// name: "from zero, invalidFromDate not zero", -// invalidFromDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// from: time.Time{}, -// want: true, -// }, -// { -// name: "invalidFromDate before from", -// invalidFromDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// from: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC), -// want: true, -// }, -// { -// name: "invalidFromDate equal to from", -// invalidFromDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// from: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// want: true, -// }, -// { -// name: "invalidFromDate after from", -// invalidFromDate: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC), -// from: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// want: false, -// }, -// } -// -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got := shouldClearInvalidState(tt.invalidFromDate, tt.from) -// assert.Equal(t, tt.want, got) -// }) -// } -//} -// -//func Test_getDeleteInvalidDate(t *testing.T) { -// tests := []struct { -// name string -// from time.Time -// invalidFromDate time.Time -// want time.Time -// }{ -// { -// name: "both zero", -// from: time.Time{}, -// invalidFromDate: time.Time{}, -// want: time.Time{}, -// }, -// { -// name: "from zero, invalidFromDate not zero", -// from: time.Time{}, -// invalidFromDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// want: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// }, -// { -// name: "from not zero, invalidFromDate zero", -// from: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// invalidFromDate: time.Time{}, -// want: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// }, -// { -// name: "from before invalidFromDate", -// from: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// invalidFromDate: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC), -// want: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC), -// }, -// { -// name: "from after invalidFromDate", -// from: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC), -// invalidFromDate: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), -// want: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC), -// }, -// } -// -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// got := getDeleteInvalidDate(tt.from, tt.invalidFromDate) -// assert.Equal(t, tt.want, got) -// }) -// } -//} diff --git a/internal/parquet/file_root_provider.go b/internal/parquet/file_root_provider.go deleted file mode 100644 index d619d2cb..00000000 --- a/internal/parquet/file_root_provider.go +++ /dev/null @@ -1,35 +0,0 @@ -package parquet - -import ( - "fmt" - "log/slog" - "sync" - "time" -) - -// FileRootProvider provides a unique file root for parquet files -// based on the current time to the nanosecond. -// If multiple files are created in the same nanosecond, the provider will increment the time by a nanosecond -// to ensure the file root is unique. -type FileRootProvider struct { - // the last time a filename was provided - lastTime time.Time - // mutex - mutex sync.Mutex -} - -// GetFileRoot returns a unique file root for a parquet file -// format is "data__" -func (p *FileRootProvider) GetFileRoot() string { - p.mutex.Lock() - defer p.mutex.Unlock() - - now := time.Now() - if now.Sub(p.lastTime) < time.Microsecond { - slog.Debug("incrementing time") - now = now.Add(time.Microsecond) - } - p.lastTime = now - - return fmt.Sprintf("data_%s_%06d", now.Format("20060102150405"), now.Nanosecond()/1000) -} diff --git a/internal/parquet/partition_pattern.go b/internal/parquet/partition_pattern.go deleted file mode 100644 index 6f839339..00000000 --- a/internal/parquet/partition_pattern.go +++ /dev/null @@ -1,36 +0,0 @@ -package parquet - -import ( - "github.com/danwakefield/fnmatch" - "github.com/turbot/tailpipe/internal/config" -) - -// PartitionPattern represents a pattern used to match partitions. -// It consists of a table pattern and a partition pattern, both of which are -// used to match a given table and partition name. -type PartitionPattern struct { - Table string - Partition string -} - -func NewPartitionPattern(partition *config.Partition) PartitionPattern { - return PartitionPattern{ - Table: partition.TableName, - Partition: partition.ShortName, - } -} - -func PartitionMatchesPatterns(table, partition string, patterns []PartitionPattern) bool { - if len(patterns) == 0 { - return true - } - // do ANY patterns match - gotMatch := false - for _, pattern := range patterns { - if fnmatch.Match(pattern.Table, table, fnmatch.FNM_CASEFOLD) && - fnmatch.Match(pattern.Partition, partition, fnmatch.FNM_CASEFOLD) { - gotMatch = true - } - } - return gotMatch -} diff --git a/internal/parquet/schema_change_error.go b/internal/parquet/schema_change_error.go deleted file mode 100644 index ba589895..00000000 --- a/internal/parquet/schema_change_error.go +++ /dev/null @@ -1,11 +0,0 @@ -package parquet - -type ColumnSchemaChange struct { - Name string - OldType string - NewType string -} - -type SchemaChangeError struct { - ChangedColumns []ColumnSchemaChange -} diff --git a/internal/parse/decode.go b/internal/parse/decode.go index 75acb37f..62c68cbe 100644 --- a/internal/parse/decode.go +++ b/internal/parse/decode.go @@ -2,9 +2,10 @@ package parse import ( "fmt" - "github.com/zclconf/go-cty/cty/gocty" "strings" + "github.com/zclconf/go-cty/cty/gocty" + "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/turbot/pipe-fittings/v2/hclhelpers" @@ -130,7 +131,7 @@ func decodePartition(block *hcl.Block, parseCtx *ConfigParseContext, resource mo for _, attr := range attrs { switch attr.Name { case "filter": - //try to evaluate expression + // try to evaluate expression val, diags := attr.Expr.Value(parseCtx.EvalCtx) res.HandleDecodeDiags(diags) // we failed, possibly as result of dependency error - give up for now @@ -138,6 +139,15 @@ func decodePartition(block *hcl.Block, parseCtx *ConfigParseContext, resource mo return res } target.Filter = val.AsString() + case "tp_index": + // try to evaluate expression + val, diags := attr.Expr.Value(parseCtx.EvalCtx) + res.HandleDecodeDiags(diags) + // we failed, possibly as result of dependency error - give up for now + if !res.Success() { + return res + } + target.TpIndexColumn = val.AsString() default: unknownAttrs = append(unknownAttrs, attr.AsHCLAttribute()) } @@ -213,6 +223,7 @@ func decodeConnection(block *hcl.Block, parseCtx *ConfigParseContext, resource m func handleUnknownHcl(block *hcl.Block, parseCtx *ConfigParseContext, unknownAttrs []*hcl.Attribute, unknownBlocks []*hcl.Block) (*config.HclBytes, hcl.Diagnostics) { var diags hcl.Diagnostics unknown := &config.HclBytes{} + for _, attr := range unknownAttrs { // get the hcl bytes for the file hclBytes := parseCtx.FileData[block.DefRange.Filename] @@ -238,8 +249,7 @@ func handleUnknownHcl(block *hcl.Block, parseCtx *ConfigParseContext, unknownAtt func decodeSource(block *hclsyntax.Block, parseCtx *ConfigParseContext) (*config.Source, *parse.DecodeResult) { res := parse.NewDecodeResult() - source := &config.Source{} - source.Type = block.Labels[0] + source := config.NewSource(block.Labels[0]) var unknownBlocks []*hcl.Block for _, block := range block.Body.Blocks { @@ -456,8 +466,7 @@ func resourceForBlock(block *hcl.Block) (modconfig.HclResource, hcl.Diagnostics) Severity: hcl.DiagError, Summary: fmt.Sprintf("resourceForBlock called for unsupported block type %s", block.Type), Subject: hclhelpers.BlockRangePointer(block), - }, - } + }} } name := fmt.Sprintf("%s.%s", block.Type, strings.Join(block.Labels, ".")) diff --git a/internal/parse/diags.go b/internal/parse/diags.go index cc9eb110..e577a7ed 100644 --- a/internal/parse/diags.go +++ b/internal/parse/diags.go @@ -2,9 +2,10 @@ package parse import ( "fmt" + "strings" + "github.com/hashicorp/hcl/v2" "github.com/turbot/pipe-fittings/v2/error_helpers" - "strings" ) // reimplement this as the pipe fittings version raises an internal error @@ -32,6 +33,11 @@ func HclDiagsToError(prefix string, diags hcl.Diagnostics) error { if prefix != "" { prefixStr = prefix + ": " } + // If the error string contains range information on a new line, move it to the same line + if strings.Contains(res, "\n(") { + parts := strings.SplitN(res, "\n(", 2) + res = fmt.Sprintf("%s (%s", parts[0], parts[1]) + } return fmt.Errorf("%s%s", prefixStr, res) } diff --git a/internal/parse/load_config_test.go b/internal/parse/load_config_test.go index f442f9f9..d4e97ddc 100644 --- a/internal/parse/load_config_test.go +++ b/internal/parse/load_config_test.go @@ -1,155 +1,870 @@ package parse -// TODO enable and fix this test -//func TestLoadTailpipeConfig(t *testing.T) { -// type args struct { -// configPath string -// partition string -// } -// tests := []struct { -// name string -// args args -// want *config.TailpipeConfig -// wantErr bool -// }{ -// // TODO #testing add more test cases -// { -// name: "static tables", -// args: args{ -// configPath: "test_data/static_table_config", -// partition: "partition.aws_cloudtrail_log.cloudtrail_logs", -// }, -// want: &config.TailpipeConfig{ -// PluginVersions: nil, -// Partitions: map[string]*config.Partition{ -// "partition.aws_cloudtrail_log.cloudtrail_logs": {}, -// "partition.aws_vpc_flow_log.flow_logs": {}, -// }, -// }, -// -// wantErr: false, -// }, -// { -// name: "dynamic tables", -// args: args{ -// configPath: "test_data/custom_table_config", -// partition: "partition.aws_cloudtrail_log.cloudtrail_logs", -// }, -// want: &config.TailpipeConfig{ -// Partitions: map[string]*config.Partition{ -// "my_csv_log.test": { -// HclResourceImpl: modconfig.HclResourceImpl{ -// FullName: "partition.my_csv_log.test", -// ShortName: "test", -// UnqualifiedName: "my_csv_log.test", -// DeclRange: hcl.Range{ -// Filename: "test_data/custom_table_config/resources.tpc", -// Start: hcl.Pos{ -// Line: 2, -// Column: 30, -// Byte: 30, -// }, -// End: hcl.Pos{ -// Line: 10, -// Column: 2, -// Byte: 230, -// }, -// }, -// BlockType: "partition", -// }, -// TableName: "my_csv_log", -// Plugin: &plugin.Plugin{ -// Instance: "custom", -// Alias: "custom", -// Plugin: "/plugins/turbot/custom@latest", -// }, -// Source: config.Source{ -// Type: "file_system", -// Config: &config.HclBytes{ -// Hcl: []byte("extensions = [\".csv\"]\npaths = [\"/Users/kai/tailpipe_data/logs\"]"), -// Range: hclhelpers.NewRange(hcl.Range{ -// Filename: "test_data/custom_table_config/resources.tpc", -// Start: hcl.Pos{ -// Line: 4, -// Column: 9, -// Byte: 68, -// }, -// End: hcl.Pos{ -// Line: 5, -// Column: 30, -// Byte: 139, -// }, -// }), -// }, -// }, -// }, -// }, -// CustomTables: map[string]*config.Table{ -// "my_csv_log": { -// HclResourceImpl: modconfig.HclResourceImpl{ -// FullName: "partition.my_csv_log.test", -// ShortName: "test", -// UnqualifiedName: "my_csv_log.test", -// DeclRange: hcl.Range{ -// Filename: "test_data/custom_table_config/resources.tpc", -// Start: hcl.Pos{ -// Line: 2, -// Column: 30, -// Byte: 30, -// }, -// End: hcl.Pos{ -// Line: 10, -// Column: 2, -// Byte: 230, -// }, -// }, -// BlockType: "partition", -// }, -// //Mode: schema.ModePartial, -// Columns: []config.ColumnSchema{ -// { -// Name: "tp_timestamp", -// Source: utils.ToPointer("time_local"), -// }, -// { -// Name: "tp_index", -// Source: utils.ToPointer("account_id"), -// }, -// { -// Name: "org_id", -// Source: utils.ToPointer("org"), -// }, -// { -// Name: "user_id", -// Type: utils.ToPointer("varchar"), -// }, -// }, -// }, -// }, -// }, -// -// wantErr: false, -// }, -// } -// -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// tailpipeDir, er := filepath.Abs(tt.args.configPath) -// if er != nil { -// t.Errorf("failed to build absolute config filepath from %s", tt.args.configPath) -// } -// // set app_specific.InstallDir -// app_specific.InstallDir = tailpipeDir -// -// tailpipeConfig, err := parseTailpipeConfig(tt.args.configPath) -// if (err != nil) != tt.wantErr { -// t.Errorf("LoadTailpipeConfig() error = %v, wantErr %v", err, tt.wantErr) -// return -// } -// -// if !reflect.DeepEqual(tailpipeConfig, tt.want) { -// t.Errorf("LoadTailpipeConfig() = %v, want %v", tailpipeConfig, tt.want) -// } -// }) -// } -//} +import ( + "fmt" + "path/filepath" + "reflect" + "sort" + "testing" + + "github.com/hashicorp/hcl/v2" + "github.com/turbot/pipe-fittings/v2/app_specific" + "github.com/turbot/pipe-fittings/v2/hclhelpers" + "github.com/turbot/pipe-fittings/v2/modconfig" + "github.com/turbot/pipe-fittings/v2/plugin" + "github.com/turbot/pipe-fittings/v2/utils" + "github.com/turbot/pipe-fittings/v2/versionfile" + "github.com/turbot/tailpipe/internal/config" +) + +func pluginVersionsEqual(l, r map[string]*versionfile.InstalledVersion) (bool, string) { + if (l == nil) != (r == nil) { + return false, "PluginVersions presence mismatch" + } + if l == nil { + return true, "" + } + if len(l) != len(r) { + return false, fmt.Sprintf("PluginVersions length mismatch: got %d want %d", len(l), len(r)) + } + for k, v := range l { + wv, ok := r[k] + if !ok { + return false, fmt.Sprintf("PluginVersions missing key '%s' in want", k) + } + if (v == nil) != (wv == nil) { + return false, fmt.Sprintf("PluginVersions['%s'] presence mismatch", k) + } + if v != nil { + if v.Name != wv.Name { + return false, fmt.Sprintf("PluginVersions['%s'].Name mismatch: got '%s' want '%s'", k, v.Name, wv.Name) + } + if v.Version != wv.Version { + return false, fmt.Sprintf("PluginVersions['%s'].Version mismatch: got '%s' want '%s'", k, v.Version, wv.Version) + } + if v.ImageDigest != wv.ImageDigest { + return false, fmt.Sprintf("PluginVersions['%s'].ImageDigest mismatch: got '%s' want '%s'", k, v.ImageDigest, wv.ImageDigest) + } + if v.BinaryDigest != wv.BinaryDigest { + return false, fmt.Sprintf("PluginVersions['%s'].BinaryDigest mismatch: got '%s' want '%s'", k, v.BinaryDigest, wv.BinaryDigest) + } + if v.BinaryArchitecture != wv.BinaryArchitecture { + return false, fmt.Sprintf("PluginVersions['%s'].BinaryArchitecture mismatch: got '%s' want '%s'", k, v.BinaryArchitecture, wv.BinaryArchitecture) + } + if v.InstalledFrom != wv.InstalledFrom { + return false, fmt.Sprintf("PluginVersions['%s'].InstalledFrom mismatch: got '%s' want '%s'", k, v.InstalledFrom, wv.InstalledFrom) + } + if v.StructVersion != wv.StructVersion { + return false, fmt.Sprintf("PluginVersions['%s'].StructVersion mismatch: got '%d' want '%d'", k, v.StructVersion, wv.StructVersion) + } + if (v.Metadata == nil) != (wv.Metadata == nil) { + return false, fmt.Sprintf("PluginVersions['%s'].Metadata presence mismatch", k) + } + if v.Metadata != nil { + if len(v.Metadata) != len(wv.Metadata) { + return false, fmt.Sprintf("PluginVersions['%s'].Metadata length mismatch", k) + } + for mk, ma := range v.Metadata { + mb, ok := wv.Metadata[mk] + if !ok { + return false, fmt.Sprintf("PluginVersions['%s'].Metadata missing key '%s'", k, mk) + } + if len(ma) != len(mb) { + return false, fmt.Sprintf("PluginVersions['%s'].Metadata['%s'] length mismatch", k, mk) + } + maCopy, mbCopy := append([]string(nil), ma...), append([]string(nil), mb...) + sort.Strings(maCopy) + sort.Strings(mbCopy) + for i := range maCopy { + if maCopy[i] != mbCopy[i] { + return false, fmt.Sprintf("PluginVersions['%s'].Metadata['%s'][%d] mismatch: got '%s' want '%s'", k, mk, i, maCopy[i], mbCopy[i]) + } + } + } + } + } + } + return true, "" +} + +func connectionsEqual(l, r map[string]*config.TailpipeConnection) (bool, string) { + if (l == nil) != (r == nil) { + return false, "Connections presence mismatch" + } + if l == nil { + return true, "" + } + if len(l) != len(r) { + return false, fmt.Sprintf("Connections length mismatch: got %d want %d", len(l), len(r)) + } + for k, conn := range l { + wconn, ok := r[k] + if !ok { + return false, fmt.Sprintf("Connections missing key '%s' in want", k) + } + if (conn == nil) != (wconn == nil) { + return false, fmt.Sprintf("Connections['%s'] presence mismatch", k) + } + if conn != nil { + if conn.HclResourceImpl.FullName != wconn.HclResourceImpl.FullName { + return false, fmt.Sprintf("Connections['%s'].HclResourceImpl.FullName mismatch: got '%s' want '%s'", k, conn.HclResourceImpl.FullName, wconn.HclResourceImpl.FullName) + } + if conn.HclResourceImpl.ShortName != wconn.HclResourceImpl.ShortName { + return false, fmt.Sprintf("Connections['%s'].HclResourceImpl.ShortName mismatch: got '%s' want '%s'", k, conn.HclResourceImpl.ShortName, wconn.HclResourceImpl.ShortName) + } + if conn.HclResourceImpl.UnqualifiedName != wconn.HclResourceImpl.UnqualifiedName { + return false, fmt.Sprintf("Connections['%s'].HclResourceImpl.UnqualifiedName mismatch: got '%s' want '%s'", k, conn.HclResourceImpl.UnqualifiedName, wconn.HclResourceImpl.UnqualifiedName) + } + if conn.HclResourceImpl.BlockType != wconn.HclResourceImpl.BlockType { + return false, fmt.Sprintf("Connections['%s'].HclResourceImpl.BlockType mismatch: got '%s' want '%s'", k, conn.HclResourceImpl.BlockType, wconn.HclResourceImpl.BlockType) + } + if conn.Plugin != wconn.Plugin { + return false, fmt.Sprintf("Connections['%s'].Plugin mismatch: got '%s' want '%s'", k, conn.Plugin, wconn.Plugin) + } + zero := hclhelpers.Range{} + connZero := conn.HclRange == zero + wconnZero := wconn.HclRange == zero + if connZero != wconnZero { + return false, fmt.Sprintf("Connections['%s'].HclRange presence mismatch", k) + } + if !connZero && !wconnZero { + if !reflect.DeepEqual(conn.HclRange, wconn.HclRange) { + gr, wr := conn.HclRange, wconn.HclRange + return false, fmt.Sprintf("Connections['%s'].HclRange mismatch: got %s:(%d,%d,%d)-(%d,%d,%d) want %s:(%d,%d,%d)-(%d,%d,%d)", k, + gr.Filename, gr.Start.Line, gr.Start.Column, gr.Start.Byte, gr.End.Line, gr.End.Column, gr.End.Byte, + wr.Filename, wr.Start.Line, wr.Start.Column, wr.Start.Byte, wr.End.Line, wr.End.Column, wr.End.Byte) + } + } + } + } + return true, "" +} + +func customTablesEqual(l, r map[string]*config.Table) (bool, string) { + if (l == nil) != (r == nil) { + return false, "CustomTables presence mismatch" + } + if l == nil { + return true, "" + } + if len(l) != len(r) { + return false, fmt.Sprintf("CustomTables length mismatch: got %d want %d", len(l), len(r)) + } + for k, ct := range l { + wct, ok := r[k] + if !ok { + return false, fmt.Sprintf("CustomTables missing key '%s' in want", k) + } + if (ct == nil) != (wct == nil) { + return false, fmt.Sprintf("CustomTables['%s'] presence mismatch", k) + } + if ct != nil { + if ct.HclResourceImpl.FullName != wct.HclResourceImpl.FullName { + return false, fmt.Sprintf("CustomTables['%s'].HclResourceImpl.FullName mismatch: got '%s' want '%s'", k, ct.HclResourceImpl.FullName, wct.HclResourceImpl.FullName) + } + if ct.HclResourceImpl.ShortName != wct.HclResourceImpl.ShortName { + return false, fmt.Sprintf("CustomTables['%s'].HclResourceImpl.ShortName mismatch: got '%s' want '%s'", k, ct.HclResourceImpl.ShortName, wct.HclResourceImpl.ShortName) + } + if ct.HclResourceImpl.UnqualifiedName != wct.HclResourceImpl.UnqualifiedName { + return false, fmt.Sprintf("CustomTables['%s'].HclResourceImpl.UnqualifiedName mismatch: got '%s' want '%s'", k, ct.HclResourceImpl.UnqualifiedName, wct.HclResourceImpl.UnqualifiedName) + } + if ct.HclResourceImpl.BlockType != wct.HclResourceImpl.BlockType { + return false, fmt.Sprintf("CustomTables['%s'].HclResourceImpl.BlockType mismatch: got '%s' want '%s'", k, ct.HclResourceImpl.BlockType, wct.HclResourceImpl.BlockType) + } + { + zero := hcl.Range{} + aZero := ct.HclResourceImpl.DeclRange == zero + bZero := wct.HclResourceImpl.DeclRange == zero + if aZero != bZero { + return false, fmt.Sprintf("CustomTables['%s'].HclResourceImpl.DeclRange presence mismatch", k) + } + if !aZero && !bZero { + if !reflect.DeepEqual(ct.HclResourceImpl.DeclRange, wct.HclResourceImpl.DeclRange) { + gr, wr := ct.HclResourceImpl.DeclRange, wct.HclResourceImpl.DeclRange + return false, fmt.Sprintf("CustomTables['%s'].HclResourceImpl.DeclRange mismatch: got %s:(%d,%d,%d)-(%d,%d,%d) want %s:(%d,%d,%d)-(%d,%d,%d)", k, + gr.Filename, gr.Start.Line, gr.Start.Column, gr.Start.Byte, gr.End.Line, gr.End.Column, gr.End.Byte, + wr.Filename, wr.Start.Line, wr.Start.Column, wr.Start.Byte, wr.End.Line, wr.End.Column, wr.End.Byte) + } + } + } + if ct.DefaultSourceFormat != nil && wct.DefaultSourceFormat != nil { + if ct.DefaultSourceFormat.Type != wct.DefaultSourceFormat.Type { + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.Type mismatch: got '%s' want '%s'", k, ct.DefaultSourceFormat.Type, wct.DefaultSourceFormat.Type) + } + if ct.DefaultSourceFormat.PresetName != wct.DefaultSourceFormat.PresetName { + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.PresetName mismatch: got '%s' want '%s'", k, ct.DefaultSourceFormat.PresetName, wct.DefaultSourceFormat.PresetName) + } + if ct.DefaultSourceFormat.HclResourceImpl.FullName != wct.DefaultSourceFormat.HclResourceImpl.FullName { + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.HclResourceImpl.FullName mismatch: got '%s' want '%s'", k, ct.DefaultSourceFormat.HclResourceImpl.FullName, wct.DefaultSourceFormat.HclResourceImpl.FullName) + } + if ct.DefaultSourceFormat.HclResourceImpl.ShortName != wct.DefaultSourceFormat.HclResourceImpl.ShortName { + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.HclResourceImpl.ShortName mismatch: got '%s' want '%s'", k, ct.DefaultSourceFormat.HclResourceImpl.ShortName, wct.DefaultSourceFormat.HclResourceImpl.ShortName) + } + if ct.DefaultSourceFormat.HclResourceImpl.UnqualifiedName != wct.DefaultSourceFormat.HclResourceImpl.UnqualifiedName { + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.HclResourceImpl.UnqualifiedName mismatch: got '%s' want '%s'", k, ct.DefaultSourceFormat.HclResourceImpl.UnqualifiedName, wct.DefaultSourceFormat.HclResourceImpl.UnqualifiedName) + } + if ct.DefaultSourceFormat.HclResourceImpl.BlockType != wct.DefaultSourceFormat.HclResourceImpl.BlockType { + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.HclResourceImpl.BlockType mismatch: got '%s' want '%s'", k, ct.DefaultSourceFormat.HclResourceImpl.BlockType, wct.DefaultSourceFormat.HclResourceImpl.BlockType) + } + { + zero := hcl.Range{} + aZero := ct.DefaultSourceFormat.HclResourceImpl.DeclRange == zero + bZero := wct.DefaultSourceFormat.HclResourceImpl.DeclRange == zero + if aZero != bZero { + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.HclResourceImpl.DeclRange presence mismatch", k) + } + if !aZero && !bZero { + if !reflect.DeepEqual(ct.DefaultSourceFormat.HclResourceImpl.DeclRange, wct.DefaultSourceFormat.HclResourceImpl.DeclRange) { + gr, wr := ct.DefaultSourceFormat.HclResourceImpl.DeclRange, wct.DefaultSourceFormat.HclResourceImpl.DeclRange + return false, fmt.Sprintf("CustomTables['%s'].DefaultSourceFormat.HclResourceImpl.DeclRange mismatch: got %s:(%d,%d,%d)-(%d,%d,%d) want %s:(%d,%d,%d)-(%d,%d,%d)", k, + gr.Filename, gr.Start.Line, gr.Start.Column, gr.Start.Byte, gr.End.Line, gr.End.Column, gr.End.Byte, + wr.Filename, wr.Start.Line, wr.Start.Column, wr.Start.Byte, wr.End.Line, wr.End.Column, wr.End.Byte) + } + } + } + } + if len(ct.Columns) != len(wct.Columns) { + return false, fmt.Sprintf("CustomTables['%s'].Columns length mismatch: got %d want %d", k, len(ct.Columns), len(wct.Columns)) + } + for i := range ct.Columns { + ac, bc := ct.Columns[i], wct.Columns[i] + if ac.Name != bc.Name { + return false, fmt.Sprintf("CustomTables['%s'].Columns[%d].Name mismatch: got '%s' want '%s'", k, i, ac.Name, bc.Name) + } + if ac.Type != nil && bc.Type != nil && *ac.Type != *bc.Type { + return false, fmt.Sprintf("CustomTables['%s'].Columns[%d].Type mismatch: got '%s' want '%s'", k, i, *ac.Type, *bc.Type) + } + if ac.Source != nil && bc.Source != nil && *ac.Source != *bc.Source { + return false, fmt.Sprintf("CustomTables['%s'].Columns[%d].Source mismatch: got '%s' want '%s'", k, i, *ac.Source, *bc.Source) + } + if ac.Description != nil && bc.Description != nil && *ac.Description != *bc.Description { + return false, fmt.Sprintf("CustomTables['%s'].Columns[%d].Description mismatch", k, i) + } + if ac.Required != nil && bc.Required != nil && *ac.Required != *bc.Required { + return false, fmt.Sprintf("CustomTables['%s'].Columns[%d].Required mismatch", k, i) + } + if ac.NullIf != nil && bc.NullIf != nil && *ac.NullIf != *bc.NullIf { + return false, fmt.Sprintf("CustomTables['%s'].Columns[%d].NullIf mismatch", k, i) + } + if ac.Transform != nil && bc.Transform != nil && *ac.Transform != *bc.Transform { + return false, fmt.Sprintf("CustomTables['%s'].Columns[%d].Transform mismatch", k, i) + } + } + mfA := append([]string(nil), ct.MapFields...) + if len(mfA) == 0 { + mfA = []string{"*"} + } + mfB := append([]string(nil), wct.MapFields...) + if len(mfB) == 0 { + mfB = []string{"*"} + } + sort.Strings(mfA) + sort.Strings(mfB) + if len(mfA) != len(mfB) { + return false, fmt.Sprintf("CustomTables['%s'].MapFields length mismatch: got %d want %d", k, len(mfA), len(mfB)) + } + for i := range mfA { + if mfA[i] != mfB[i] { + return false, fmt.Sprintf("CustomTables['%s'].MapFields[%d] mismatch: got '%s' want '%s'", k, i, mfA[i], mfB[i]) + } + } + if ct.NullIf != wct.NullIf { + return false, fmt.Sprintf("CustomTables['%s'].NullIf mismatch: got '%s' want '%s'", k, ct.NullIf, wct.NullIf) + } + } + } + return true, "" +} + +func formatsEqual(l, r map[string]*config.Format) (bool, string) { + if (l == nil) != (r == nil) { + return false, "Formats presence mismatch" + } + if l == nil { + return true, "" + } + if len(l) != len(r) { + return false, fmt.Sprintf("Formats length mismatch: got %d want %d", len(l), len(r)) + } + for k, f := range l { + wf, ok := r[k] + if !ok { + return false, fmt.Sprintf("Formats missing key '%s' in want", k) + } + if (f == nil) != (wf == nil) { + return false, fmt.Sprintf("Formats['%s'] presence mismatch", k) + } + if f != nil { + if f.Type != wf.Type { + return false, fmt.Sprintf("Formats['%s'].Type mismatch: got '%s' want '%s'", k, f.Type, wf.Type) + } + if f.HclResourceImpl.FullName != wf.HclResourceImpl.FullName { + return false, fmt.Sprintf("Formats['%s'].HclResourceImpl.FullName mismatch: got '%s' want '%s'", k, f.HclResourceImpl.FullName, wf.HclResourceImpl.FullName) + } + if f.HclResourceImpl.ShortName != wf.HclResourceImpl.ShortName { + return false, fmt.Sprintf("Formats['%s'].HclResourceImpl.ShortName mismatch: got '%s' want '%s'", k, f.HclResourceImpl.ShortName, wf.HclResourceImpl.ShortName) + } + if f.HclResourceImpl.UnqualifiedName != wf.HclResourceImpl.UnqualifiedName { + return false, fmt.Sprintf("Formats['%s'].HclResourceImpl.UnqualifiedName mismatch: got '%s' want '%s'", k, f.HclResourceImpl.UnqualifiedName, wf.HclResourceImpl.UnqualifiedName) + } + if f.HclResourceImpl.BlockType != wf.HclResourceImpl.BlockType { + return false, fmt.Sprintf("Formats['%s'].HclResourceImpl.BlockType mismatch: got '%s' want '%s'", k, f.HclResourceImpl.BlockType, wf.HclResourceImpl.BlockType) + } + { + zero := hcl.Range{} + aZero := f.HclResourceImpl.DeclRange == zero + bZero := wf.HclResourceImpl.DeclRange == zero + if aZero != bZero { + return false, fmt.Sprintf("Formats['%s'].HclResourceImpl.DeclRange presence mismatch", k) + } + if !aZero && !bZero { + if !reflect.DeepEqual(f.HclResourceImpl.DeclRange, wf.HclResourceImpl.DeclRange) { + gr, wr := f.HclResourceImpl.DeclRange, wf.HclResourceImpl.DeclRange + return false, fmt.Sprintf("Formats['%s'].HclResourceImpl.DeclRange mismatch: got %s:(%d,%d,%d)-(%d,%d,%d) want %s:(%d,%d,%d)-(%d,%d,%d)", k, + gr.Filename, gr.Start.Line, gr.Start.Column, gr.Start.Byte, gr.End.Line, gr.End.Column, gr.End.Byte, + wr.Filename, wr.Start.Line, wr.Start.Column, wr.Start.Byte, wr.End.Line, wr.End.Column, wr.End.Byte) + } + } + } + if f.PresetName != "" && wf.PresetName != "" && f.PresetName != wf.PresetName { + return false, fmt.Sprintf("Formats['%s'].PresetName mismatch: got '%s' want '%s'", k, f.PresetName, wf.PresetName) + } + } + } + return true, "" +} + +func partitionsEqual(l, r map[string]*config.Partition) (bool, string) { + if (l == nil) != (r == nil) { + return false, "Partitions presence mismatch" + } + if l == nil { + return true, "" + } + if len(l) != len(r) { + return false, fmt.Sprintf("Partitions length mismatch: got %d want %d", len(l), len(r)) + } + for k, p := range l { + wp, ok := r[k] + if !ok { + return false, fmt.Sprintf("Partitions missing key '%s' in want", k) + } + if (p == nil) != (wp == nil) { + return false, fmt.Sprintf("Partitions['%s'] presence mismatch", k) + } + if p != nil { + if p.HclResourceImpl.FullName != wp.HclResourceImpl.FullName { + return false, fmt.Sprintf("Partitions['%s'].HclResourceImpl.FullName mismatch: got '%s' want '%s'", k, p.HclResourceImpl.FullName, wp.HclResourceImpl.FullName) + } + if p.HclResourceImpl.ShortName != wp.HclResourceImpl.ShortName { + return false, fmt.Sprintf("Partitions['%s'].HclResourceImpl.ShortName mismatch: got '%s' want '%s'", k, p.HclResourceImpl.ShortName, wp.HclResourceImpl.ShortName) + } + if p.HclResourceImpl.UnqualifiedName != wp.HclResourceImpl.UnqualifiedName { + return false, fmt.Sprintf("Partitions['%s'].HclResourceImpl.UnqualifiedName mismatch: got '%s' want '%s'", k, p.HclResourceImpl.UnqualifiedName, wp.HclResourceImpl.UnqualifiedName) + } + if p.HclResourceImpl.BlockType != wp.HclResourceImpl.BlockType { + return false, fmt.Sprintf("Partitions['%s'].HclResourceImpl.BlockType mismatch: got '%s' want '%s'", k, p.HclResourceImpl.BlockType, wp.HclResourceImpl.BlockType) + } + { + zero := hcl.Range{} + aZero := p.HclResourceImpl.DeclRange == zero + bZero := wp.HclResourceImpl.DeclRange == zero + if aZero != bZero { + return false, fmt.Sprintf("Partitions['%s'].HclResourceImpl.DeclRange presence mismatch", k) + } + if !aZero && !bZero { + if !reflect.DeepEqual(p.HclResourceImpl.DeclRange, wp.HclResourceImpl.DeclRange) { + gr, wr := p.HclResourceImpl.DeclRange, wp.HclResourceImpl.DeclRange + return false, fmt.Sprintf("Partitions['%s'].HclResourceImpl.DeclRange mismatch: got %s:(%d,%d,%d)-(%d,%d,%d) want %s:(%d,%d,%d)-(%d,%d,%d)", k, + gr.Filename, gr.Start.Line, gr.Start.Column, gr.Start.Byte, gr.End.Line, gr.End.Column, gr.End.Byte, + wr.Filename, wr.Start.Line, wr.Start.Column, wr.Start.Byte, wr.End.Line, wr.End.Column, wr.End.Byte) + } + } + } + if p.TableName != wp.TableName { + return false, fmt.Sprintf("Partitions['%s'].TableName mismatch: got '%s' want '%s'", k, p.TableName, wp.TableName) + } + if p.Source.Type != wp.Source.Type { + return false, fmt.Sprintf("Partitions['%s'].Source.Type mismatch: got '%s' want '%s'", k, p.Source.Type, wp.Source.Type) + } + if (p.Source.Connection == nil) != (wp.Source.Connection == nil) { + return false, fmt.Sprintf("Partitions['%s'].Source.Connection presence mismatch", k) + } + if p.Source.Connection != nil && wp.Source.Connection != nil { + if p.Source.Connection.HclResourceImpl.UnqualifiedName != wp.Source.Connection.HclResourceImpl.UnqualifiedName { + return false, fmt.Sprintf("Partitions['%s'].Source.Connection.HclResourceImpl.UnqualifiedName mismatch: got '%s' want '%s'", k, p.Source.Connection.HclResourceImpl.UnqualifiedName, wp.Source.Connection.HclResourceImpl.UnqualifiedName) + } + } + if (p.Source.Format == nil) != (wp.Source.Format == nil) { + return false, fmt.Sprintf("Partitions['%s'].Source.Format presence mismatch", k) + } + if p.Source.Format != nil && wp.Source.Format != nil { + pf, of := p.Source.Format, wp.Source.Format + if pf.Type != of.Type { + return false, fmt.Sprintf("Partitions['%s'].Source.Format.Type mismatch: got '%s' want '%s'", k, pf.Type, of.Type) + } + if pf.PresetName != of.PresetName { + return false, fmt.Sprintf("Partitions['%s'].Source.Format.PresetName mismatch: got '%s' want '%s'", k, pf.PresetName, of.PresetName) + } + if pf.HclResourceImpl.FullName != of.HclResourceImpl.FullName { + return false, fmt.Sprintf("Partitions['%s'].Source.Format.HclResourceImpl.FullName mismatch: got '%s' want '%s'", k, pf.HclResourceImpl.FullName, of.HclResourceImpl.FullName) + } + if pf.HclResourceImpl.ShortName != of.HclResourceImpl.ShortName { + return false, fmt.Sprintf("Partitions['%s'].Source.Format.HclResourceImpl.ShortName mismatch: got '%s' want '%s'", k, pf.HclResourceImpl.ShortName, of.HclResourceImpl.ShortName) + } + if pf.HclResourceImpl.UnqualifiedName != of.HclResourceImpl.UnqualifiedName { + return false, fmt.Sprintf("Partitions['%s'].Source.Format.HclResourceImpl.UnqualifiedName mismatch: got '%s' want '%s'", k, pf.HclResourceImpl.UnqualifiedName, of.HclResourceImpl.UnqualifiedName) + } + if pf.HclResourceImpl.BlockType != of.HclResourceImpl.BlockType { + return false, fmt.Sprintf("Partitions['%s'].Source.Format.HclResourceImpl.BlockType mismatch: got '%s' want '%s'", k, pf.HclResourceImpl.BlockType, of.HclResourceImpl.BlockType) + } + } + if (p.Source.Config == nil) != (wp.Source.Config == nil) { + return false, fmt.Sprintf("Partitions['%s'].Source.Config presence mismatch", k) + } + if p.Source.Config != nil && p.Source.Config.Range != wp.Source.Config.Range { + return false, fmt.Sprintf("Partitions['%s'].Source.Config.Range mismatch", k) + } + if !(len(p.Config) == 0 && len(wp.Config) == 0) { + if string(p.Config) != string(wp.Config) { + return false, fmt.Sprintf("Partitions['%s'].Config bytes mismatch", k) + } + if p.ConfigRange != wp.ConfigRange { + return false, fmt.Sprintf("Partitions['%s'].ConfigRange mismatch", k) + } + } + if p.Filter != wp.Filter || p.TpIndexColumn != wp.TpIndexColumn { + return false, fmt.Sprintf("Partitions['%s'].Filter/TpIndexColumn mismatch", k) + } + if (p.CustomTable == nil) != (wp.CustomTable == nil) { + return false, fmt.Sprintf("Partitions['%s'].CustomTable presence mismatch", k) + } + if p.CustomTable != nil && wp.CustomTable != nil { + if !reflect.DeepEqual(p.CustomTable, wp.CustomTable) { + return false, fmt.Sprintf("Partitions['%s'].CustomTable mismatch", k) + } + } + if p.Plugin != nil && wp.Plugin != nil { + if p.Plugin.Instance != wp.Plugin.Instance { + return false, fmt.Sprintf("Partitions['%s'].Plugin.Instance mismatch: got '%s' want '%s'", k, p.Plugin.Instance, wp.Plugin.Instance) + } + if p.Plugin.Alias != wp.Plugin.Alias { + return false, fmt.Sprintf("Partitions['%s'].Plugin.Alias mismatch: got '%s' want '%s'", k, p.Plugin.Alias, wp.Plugin.Alias) + } + if p.Plugin.Plugin != wp.Plugin.Plugin { + return false, fmt.Sprintf("Partitions['%s'].Plugin.Plugin mismatch: got '%s' want '%s'", k, p.Plugin.Plugin, wp.Plugin.Plugin) + } + } + } + } + return true, "" +} + +func tailpipeConfigEqual(l, r *config.TailpipeConfig) (bool, string) { + if l == nil || r == nil { + if l == r { + return true, "" + } + return false, "nil vs non-nil TailpipeConfig" + } + if ok, msg := pluginVersionsEqual(l.PluginVersions, r.PluginVersions); !ok { + return false, msg + } + if ok, msg := partitionsEqual(l.Partitions, r.Partitions); !ok { + return false, msg + } + if ok, msg := connectionsEqual(l.Connections, r.Connections); !ok { + return false, msg + } + if ok, msg := customTablesEqual(l.CustomTables, r.CustomTables); !ok { + return false, msg + } + if ok, msg := formatsEqual(l.Formats, r.Formats); !ok { + return false, msg + } + return true, "" +} + +func TestParseTailpipeConfig(t *testing.T) { + type args struct { + configPath string + partition string + } + tests := []struct { + name string + args args + want *config.TailpipeConfig + wantErr bool + }{ + { + name: "static tables", + args: args{ + configPath: "test_data/static_table_config", + partition: "partition.aws_cloudtrail_log.cloudtrail_logs", + }, + want: &config.TailpipeConfig{ + PluginVersions: map[string]*versionfile.InstalledVersion{}, + Partitions: map[string]*config.Partition{ + "aws_cloudtrail_log.cloudtrail_logs": { + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "aws_cloudtrail_log.cloudtrail_logs", + ShortName: "cloudtrail_logs", + UnqualifiedName: "aws_cloudtrail_log.cloudtrail_logs", + DeclRange: hcl.Range{ + Filename: "test_data/static_table_config/resources.tpc", + Start: hcl.Pos{Line: 3, Column: 50, Byte: 103}, + End: hcl.Pos{Line: 9, Column: 2, Byte: 252}, + }, + BlockType: "partition", + }, + TableName: "aws_cloudtrail_log", + Source: config.Source{ + Type: "file_system", + Config: &config.HclBytes{ + Hcl: []byte("extensions = [\".csv\"]\npaths = [\"/Users/kai/tailpipe_data/logs\"]"), + Range: hclhelpers.NewRange(hcl.Range{ + Filename: "test_data/static_table_config/resources.tpc", + Start: hcl.Pos{ + Line: 6, + Column: 6, + Byte: 157, + }, + End: hcl.Pos{ + Line: 7, + Column: 29, + Byte: 244, + }, + }), + }, + }, + Config: []byte(" plugin = \"aws\"\n"), + ConfigRange: hclhelpers.NewRange(hcl.Range{ + Filename: "test_data/static_table_config/resources.tpc", + Start: hcl.Pos{ + Line: 4, + Column: 5, + Byte: 109, + }, + End: hcl.Pos{ + Line: 4, + Column: 19, + Byte: 123, + }, + }), + }, + "aws_vpc_flow_log.flow_logs": { + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "aws_vpc_flow_log.flow_logs", + ShortName: "flow_logs", + UnqualifiedName: "aws_vpc_flow_log.flow_logs", + DeclRange: hcl.Range{ + Filename: "test_data/static_table_config/resources.tpc", + Start: hcl.Pos{Line: 12, Column: 42, Byte: 351}, + End: hcl.Pos{Line: 22, Column: 2, Byte: 636}, + }, + BlockType: "partition", + }, + TableName: "aws_vpc_flow_log", + Source: config.Source{ + Type: "aws_cloudwatch", + Config: &config.HclBytes{ + Hcl: []byte( + "log_group_name = \"/victor/vpc/flowlog\"\n" + + "start_time = \"2024-08-12T07:56:26Z\"\n" + + "end_time = \"2024-08-13T07:56:26Z\"\n" + + "access_key = \"REPLACE\"\n" + + "secret_key = \"REPLACE\"\n" + + "session_token = \"REPLACE\"", + ), + Range: hclhelpers.NewRange(hcl.Range{ + Filename: "test_data/static_table_config/resources.tpc", + Start: hcl.Pos{Line: 15, Column: 6, Byte: 408}, + End: hcl.Pos{Line: 20, Column: 34, Byte: 628}, + }), + }, + }, + // Unknown attr captured at partition level + Config: []byte(" plugin = \"aws\"\n"), + ConfigRange: hclhelpers.NewRange(hcl.Range{ + Filename: "test_data/static_table_config/resources.tpc", + Start: hcl.Pos{Line: 13, Column: 5, Byte: 357}, + End: hcl.Pos{Line: 13, Column: 19, Byte: 371}, + }), + }, + }, + Connections: map[string]*config.TailpipeConnection{}, + CustomTables: map[string]*config.Table{}, + Formats: map[string]*config.Format{}, + }, + + wantErr: false, + }, + { + name: "dynamic tables", + args: args{ + configPath: "test_data/custom_table_config", + }, + want: &config.TailpipeConfig{ + Partitions: map[string]*config.Partition{ + "my_csv_log.test": { + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "my_csv_log.test", + ShortName: "test", + UnqualifiedName: "my_csv_log.test", + DeclRange: hcl.Range{ + Filename: "test_data/custom_table_config/resources.tpc", + Start: hcl.Pos{ + Line: 2, + Column: 30, + Byte: 30, + }, + End: hcl.Pos{ + Line: 10, + Column: 2, + Byte: 239, + }, + }, + BlockType: "partition", + }, + TableName: "my_csv_log", + Plugin: &plugin.Plugin{ + Instance: "custom", + Alias: "custom", + Plugin: "/plugins/turbot/custom@latest", + }, + Source: config.Source{ + Type: "file_system", + Format: &config.Format{ + Type: "delimited", + PresetName: "", + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "delimited.csv_logs", + ShortName: "csv_logs", + UnqualifiedName: "delimited.csv_logs", + BlockType: "format", + }, + }, + Config: &config.HclBytes{ + Hcl: []byte("extensions = [\".csv\"]\npaths = [\"/Users/kai/tailpipe_data/logs\"]"), + Range: hclhelpers.NewRange(hcl.Range{ + Filename: "test_data/custom_table_config/resources.tpc", + Start: hcl.Pos{ + Line: 4, + Column: 9, + Byte: 68, + }, + End: hcl.Pos{ + Line: 5, + Column: 30, + Byte: 139, + }, + }), + }, + }, + }, + }, + CustomTables: map[string]*config.Table{ + "my_csv_log": { + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "table.my_csv_log", + ShortName: "my_csv_log", + UnqualifiedName: "my_csv_log", + DeclRange: hcl.Range{ + Filename: "test_data/custom_table_config/resources.tpc", + Start: hcl.Pos{ + Line: 14, + Column: 21, + Byte: 295, + }, + End: hcl.Pos{ + Line: 29, + Column: 2, + Byte: 602, + }, + }, + BlockType: "table", + }, + //Mode: schema.ModePartial, + Columns: []config.Column{ + { + Name: "tp_timestamp", + Source: utils.ToPointer("time_local"), + }, + { + Name: "tp_index", + Source: utils.ToPointer("account_id"), + }, + { + Name: "org_id", + Source: utils.ToPointer("org"), + }, + { + Name: "user_id", + Type: utils.ToPointer("varchar"), + }, + }, + }, + }, + Connections: map[string]*config.TailpipeConnection{}, + Formats: map[string]*config.Format{ + "delimited.csv_default_logs": { + Type: "delimited", + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "delimited.csv_default_logs", + ShortName: "csv_default_logs", + UnqualifiedName: "delimited.csv_default_logs", + DeclRange: hcl.Range{ + Filename: "test_data/custom_table_config/resources.tpc", + Start: hcl.Pos{ + Line: 33, + Column: 39, + Byte: 644, + }, + End: hcl.Pos{ + Line: 35, + Column: 2, + Byte: 648, + }, + }, + BlockType: "format", + }, + }, + "delimited.csv_logs": { + Type: "delimited", + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "delimited.csv_logs", + ShortName: "csv_logs", + UnqualifiedName: "delimited.csv_logs", + DeclRange: hcl.Range{ + Filename: "test_data/custom_table_config/resources.tpc", + Start: hcl.Pos{ + Line: 37, + Column: 32, + Byte: 681, + }, + End: hcl.Pos{ + Line: 40, + Column: 2, + Byte: 743, + }, + }, + BlockType: "format", + }, + Config: &config.HclBytes{ + Hcl: []byte( + " header = false\n\n delimiter = \"\\t\"\n", + ), + Range: hclhelpers.NewRange(hcl.Range{ + Filename: "test_data/static_table_config/resources.tpc", + Start: hcl.Pos{Line: 38, Column: 5, Byte: 687}, + End: hcl.Pos{Line: 39, Column: 30, Byte: 741}, + }), + }, + }, + }, + PluginVersions: map[string]*versionfile.InstalledVersion{}, + }, + + wantErr: false, + }, + { + name: "invalid path", + args: args{ + configPath: "test_data/does_not_exist", + }, + want: &config.TailpipeConfig{ + PluginVersions: map[string]*versionfile.InstalledVersion{}, + Partitions: map[string]*config.Partition{}, + Connections: map[string]*config.TailpipeConnection{}, + CustomTables: map[string]*config.Table{}, + Formats: map[string]*config.Format{}, + }, + wantErr: false, + }, + { + name: "malformed hcl", + args: args{ + configPath: "test_data/malformed_config", + }, + want: nil, + wantErr: true, + }, + { + name: "invalid partition labels", + args: args{ + configPath: "test_data/invalid_partition_labels", + }, + want: nil, + wantErr: true, + }, + { + name: "connections config", + args: args{ + configPath: "test_data/connections_config", + }, + want: &config.TailpipeConfig{ + PluginVersions: map[string]*versionfile.InstalledVersion{}, + Partitions: map[string]*config.Partition{ + "aws_alb_connection_log.aws_alb_connection_log": { + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "aws_alb_connection_log.aws_alb_connection_log", + ShortName: "aws_alb_connection_log", + UnqualifiedName: "aws_alb_connection_log.aws_alb_connection_log", + DeclRange: hcl.Range{Filename: "test_data/connections_config/resources.tpc", Start: hcl.Pos{Line: 8, Column: 61, Byte: 155}, End: hcl.Pos{Line: 13, Column: 2, Byte: 278}}, + BlockType: "partition", + }, + TableName: "aws_alb_connection_log", + Source: config.Source{ + Type: "aws_s3_bucket", + Connection: &config.TailpipeConnection{ + HclResourceImpl: modconfig.HclResourceImpl{UnqualifiedName: "aws.primary"}, + }, + Config: &config.HclBytes{ + Range: hclhelpers.NewRange(hcl.Range{Filename: "test_data/connections_config/resources.tpc", Start: hcl.Pos{Line: 11, Column: 5, Byte: 228}, End: hcl.Pos{Line: 11, Column: 49, Byte: 272}}), + }, + }, + }, + }, + Connections: map[string]*config.TailpipeConnection{ + "aws.primary": { + HclResourceImpl: modconfig.HclResourceImpl{ + FullName: "aws.primary", + ShortName: "primary", + UnqualifiedName: "aws.primary", + BlockType: "connection", + }, + Plugin: "aws", + HclRange: hclhelpers.NewRange(hcl.Range{Filename: "test_data/connections_config/resources.tpc", Start: hcl.Pos{Line: 2, Column: 3, Byte: 31}, End: hcl.Pos{Line: 4, Column: 23, Byte: 90}}), + }, + }, + CustomTables: map[string]*config.Table{}, + Formats: map[string]*config.Format{}, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tailpipeDir, er := filepath.Abs(tt.args.configPath) + if er != nil { + t.Errorf("failed to build absolute config filepath from %s", tt.args.configPath) + } + // set app_specific.InstallDir + app_specific.InstallDir = tailpipeDir + + tailpipeConfig, err := parseTailpipeConfig(tt.args.configPath) + if (err.Error != nil) != tt.wantErr { + t.Errorf("LoadTailpipeConfig() error = %v, wantErr %v", err.Error, tt.wantErr) + return + } + + // use TailpipeConfig.EqualConfig for all cases (ignores Source.Config.Hcl differences) + if ok, msg := tailpipeConfigEqual(tailpipeConfig, tt.want); !ok { + t.Errorf("TailpipeConfig mismatch: %s", msg) + return + } + + }) + } +} diff --git a/internal/parse/test_data/connections_config/resources.tpc b/internal/parse/test_data/connections_config/resources.tpc new file mode 100644 index 00000000..7873c440 --- /dev/null +++ b/internal/parse/test_data/connections_config/resources.tpc @@ -0,0 +1,13 @@ +connection "aws" "primary" { + profile = "primary" + plugin = "aws" + region = "us-east-1" +} + + +partition "aws_alb_connection_log" "aws_alb_connection_log" { + source "aws_s3_bucket" { + connection = connection.aws.primary + bucket = "alb-connection-logs-test-tailpipe" + } +} diff --git a/internal/parse/test_data/custom_table_config/resources.tpc b/internal/parse/test_data/custom_table_config/resources.tpc index 48f2abcd..9e6a0b19 100644 --- a/internal/parse/test_data/custom_table_config/resources.tpc +++ b/internal/parse/test_data/custom_table_config/resources.tpc @@ -5,14 +5,14 @@ partition "my_csv_log" "test"{ extensions = [".csv"] # format MUST be set for a custom table - format = format.csv_logs + format = format.delimited.csv_logs } } # define a custom table 'my_log' table "my_csv_log" { - format = format.csv_default_logs + format = format.delimited.csv_default_logs # the partition to use column "tp_timestamp" { source = "time_local" diff --git a/internal/parse/test_data/invalid_partition_labels/resources.tpc b/internal/parse/test_data/invalid_partition_labels/resources.tpc new file mode 100644 index 00000000..4a2ec632 --- /dev/null +++ b/internal/parse/test_data/invalid_partition_labels/resources.tpc @@ -0,0 +1,4 @@ +partition my_csv_log { + # missing 2nd label + source file_system { paths = ["/tmp"] } +} diff --git a/internal/parse/test_data/malformed_config/resources.tpc b/internal/parse/test_data/malformed_config/resources.tpc new file mode 100644 index 00000000..ec27ba1f --- /dev/null +++ b/internal/parse/test_data/malformed_config/resources.tpc @@ -0,0 +1,5 @@ +partition aws_cloudtrail_log cloudtrail_logs { + source file_system { + paths = ["/tmp"] + } + # missing closing brace here intentionally diff --git a/internal/plugin/collect_response.go b/internal/plugin/collect_response.go index 11c2e8eb..e428199b 100644 --- a/internal/plugin/collect_response.go +++ b/internal/plugin/collect_response.go @@ -13,9 +13,14 @@ type CollectResponse struct { } func CollectResponseFromProto(resp *proto.CollectResponse) *CollectResponse { + // tactical - because up until sdk v0.8.0, the returned schema was not merged with the common schema, + // we need to merge it here + // otherwise the `required` property for common fields will not be set + s := schema.TableSchemaFromProto(resp.Schema).MergeWithCommonSchema() + return &CollectResponse{ ExecutionId: resp.ExecutionId, - Schema: schema.TableSchemaFromProto(resp.Schema), + Schema: s, FromTime: row_source.ResolvedFromTimeFromProto(resp.FromTime), } } diff --git a/internal/plugin/errors.go b/internal/plugin/errors.go new file mode 100644 index 00000000..ce534b08 --- /dev/null +++ b/internal/plugin/errors.go @@ -0,0 +1,28 @@ +package plugin + +import ( + "errors" + "strings" +) + +// cleanupPluginError is a utility function to clean up errors returned the plugin event stream +// and make them more user-friendly. It handles specific error cases, such as RPC errors and in particular +// EOF errors caused by plugin crashes, and transforms them into more meaningful messages. +func cleanupPluginError(err error) error { + if err == nil { + return nil + } + + errString := strings.TrimSpace(err.Error()) + + // if this is an RPC Error while talking with the plugin + if strings.HasPrefix(errString, "rpc error") { + if strings.Contains(errString, "error reading from server: EOF") { + errString = "lost connection to plugin" + } else { + // trim out "rpc error: code = Unknown desc =" + errString = strings.TrimPrefix(errString, "rpc error: code = Unknown desc =") + } + } + return errors.New(strings.TrimSpace(errString)) +} diff --git a/internal/plugin/installation_actions.go b/internal/plugin/installation_actions.go index 6bd2c82b..3c413917 100644 --- a/internal/plugin/installation_actions.go +++ b/internal/plugin/installation_actions.go @@ -116,7 +116,8 @@ func List(ctx context.Context, pluginVersions map[string]*versionfile.InstalledV // detectLocalPlugin returns true if the modTime of the `pluginBinary` is after the installation date as recorded in the installation data // this may happen when a plugin is installed from the registry, but is then compiled from source func detectLocalPlugin(installation *versionfile.InstalledVersion, pluginBinary string) bool { - // TODO this should no longer be necessary as we now have a "local" version number in the versions file? + // Guard: newly developed local plugins may not have a versions entry yet, + // so installation can be nil. Keep this check to prevent a nil-pointer dereference. if installation == nil { return true } diff --git a/internal/plugin/observer.go b/internal/plugin/observer.go deleted file mode 100644 index 83fba8cf..00000000 --- a/internal/plugin/observer.go +++ /dev/null @@ -1,9 +0,0 @@ -package plugin - -import ( - "github.com/turbot/tailpipe-plugin-sdk/grpc/proto" -) - -type Observer interface { - Notify(event *proto.Event) -} diff --git a/internal/plugin/plugin_manager.go b/internal/plugin/plugin_manager.go index 8ea1f441..2a619bfc 100644 --- a/internal/plugin/plugin_manager.go +++ b/internal/plugin/plugin_manager.go @@ -9,17 +9,20 @@ import ( "os/exec" "path/filepath" "strconv" - "strings" "sync" "time" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "github.com/hashicorp/go-hclog" goplugin "github.com/hashicorp/go-plugin" "github.com/hashicorp/go-version" _ "github.com/marcboeker/go-duckdb/v2" - "github.com/turbot/go-kit/helpers" + "github.com/spf13/viper" + gokithelpers "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/app_specific" - "github.com/turbot/pipe-fittings/v2/error_helpers" + pconstants "github.com/turbot/pipe-fittings/v2/constants" pfilepaths "github.com/turbot/pipe-fittings/v2/filepaths" "github.com/turbot/pipe-fittings/v2/installationstate" pociinstaller "github.com/turbot/pipe-fittings/v2/ociinstaller" @@ -27,12 +30,16 @@ import ( "github.com/turbot/pipe-fittings/v2/statushooks" "github.com/turbot/pipe-fittings/v2/versionfile" "github.com/turbot/tailpipe-plugin-core/core" + "github.com/turbot/tailpipe-plugin-sdk/events" "github.com/turbot/tailpipe-plugin-sdk/grpc" "github.com/turbot/tailpipe-plugin-sdk/grpc/proto" "github.com/turbot/tailpipe-plugin-sdk/grpc/shared" + "github.com/turbot/tailpipe-plugin-sdk/observable" "github.com/turbot/tailpipe-plugin-sdk/types" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/constants" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" + "github.com/turbot/tailpipe/internal/helpers" "github.com/turbot/tailpipe/internal/ociinstaller" "google.golang.org/protobuf/types/known/timestamppb" @@ -44,8 +51,10 @@ type PluginManager struct { // map of running plugins, keyed by plugin name Plugins map[string]*grpc.PluginClient pluginMutex sync.RWMutex - obs Observer - pluginPath string + // the observer to notify of events + // (this will be the collector) + obs observable.Observer + pluginPath string } func NewPluginManager() *PluginManager { @@ -57,12 +66,12 @@ func NewPluginManager() *PluginManager { // AddObserver adds a // n observer to the plugin manager -func (p *PluginManager) AddObserver(o Observer) { +func (p *PluginManager) AddObserver(o observable.Observer) { p.obs = o } // Collect starts the plugin if needed, discovers the artifacts and download them for the given partition. -func (p *PluginManager) Collect(ctx context.Context, partition *config.Partition, fromTime time.Time, collectionTempDir string) (*CollectResponse, error) { +func (p *PluginManager) Collect(ctx context.Context, partition *config.Partition, fromTime time.Time, toTime time.Time, recollect bool, collectionTempDir string) (*CollectResponse, error) { // start plugin if needed tablePlugin := partition.Plugin tablePluginClient, err := p.getPlugin(tablePlugin) @@ -102,6 +111,9 @@ func (p *PluginManager) Collect(ctx context.Context, partition *config.Partition CollectionStatePath: collectionStatePath, SourceData: partition.Source.ToProto(), FromTime: timestamppb.New(fromTime), + ToTime: timestamppb.New(toTime), + TempDirMaxMb: viper.GetInt64(pconstants.ArgTempDirMaxMb), + Recollect: &recollect, } if partition.Source.Connection != nil { @@ -110,13 +122,19 @@ func (p *PluginManager) Collect(ctx context.Context, partition *config.Partition // identify which plugin provides the source and if it is different from the table plugin, // we need to start the source plugin, and then pass reattach info - sourcePluginReattach, err := p.getSourcePluginReattach(ctx, partition, tablePlugin) + sourcePluginClient, sourcePluginReattach, err := p.getSourcePluginReattach(ctx, partition, tablePlugin) if err != nil { return nil, err } // set on req (may be nil - this is fine) req.SourcePlugin = sourcePluginReattach + err = p.verifySupportedOperations(tablePluginClient, sourcePluginClient) + if err != nil { + return nil, err + } + + // start a goroutine to monitor the plugins // populate the custom table if partition.CustomTable != nil { req.CustomTableSchema = partition.CustomTable.ToProto() @@ -134,19 +152,75 @@ func (p *PluginManager) Collect(ctx context.Context, partition *config.Partition collectResponse, err := tablePluginClient.Collect(req) if err != nil { - return nil, fmt.Errorf("error starting collection for plugin %s: %w", tablePluginClient.Name, error_helpers.TransformErrorToSteampipe(err)) + return nil, fmt.Errorf("error starting collection for plugin %s: %w", tablePluginClient.Name, error_helpers.TransformErrorToTailpipe(err)) } // start a goroutine to read the eventStream and listen to file events // this will loop until it hits an error or the stream is closed - go p.readCollectionEvents(ctx, eventStream) + go p.readCollectionEvents(ctx, executionID, eventStream) // just return - the observer is responsible for waiting for completion return CollectResponseFromProto(collectResponse), nil } +// verifySupportedOperations checks if the table plugin and source plugin (if different) support time ranges +// if they do not support time ranges set the 'Overwrite' flag to true and if the 'To' time is set, return an error +func (p *PluginManager) verifySupportedOperations(tablePluginClient *grpc.PluginClient, sourcePluginClient *grpc.PluginClient) error { + tablePluginSupportedOperations, err := p.getSupportedOperations(tablePluginClient) + if err != nil { + return fmt.Errorf("error getting supported operations for plugin %s: %w", tablePluginClient.Name, err) + } + tablePluginName := pociinstaller.NewImageRef(tablePluginClient.Name).GetFriendlyName() + + var sourcePluginSupportedOperations *proto.GetSupportedOperationsResponse + var sourcePluginName string + if sourcePluginClient != nil { + sourcePluginSupportedOperations, err = p.getSupportedOperations(sourcePluginClient) + if err != nil { + return fmt.Errorf("error getting supported operations for plugin %s: %w", tablePluginClient.Name, err) + } + sourcePluginName = pociinstaller.NewImageRef(sourcePluginClient.Name).GetFriendlyName() + } + + // if the plugin does not support time ranges: + // - we cannot specify a 'To' time + // - hard code recollect to true - this is the default behaviour for plugins that do not support time ranges + // if a 'To' time' is set, we must ensure the plugin supports time ranges + if !tablePluginSupportedOperations.TimeRanges { + slog.Info("plugin does not support time ranges - setting 'Overwrite' to true", "plugin", tablePluginName) + viper.Set(pconstants.ArgOverwrite, true) + + if viper.IsSet(pconstants.ArgTo) { + return fmt.Errorf("plugin '%s' does not support specifying a 'To' time - try updating the plugin", tablePluginName) + } + } + + if sourcePluginSupportedOperations != nil && !sourcePluginSupportedOperations.TimeRanges { + slog.Info("plugin does not support time ranges - setting 'Overwrite' to true", "plugin", sourcePluginName) + viper.Set(pconstants.ArgOverwrite, true) + + if viper.IsSet(pconstants.ArgTo) { + return fmt.Errorf("source plugin '%s' does not support specifying a 'To' time - try updating the plugin", sourcePluginName) + } + } + return nil +} + +// getSupportedOperations calls the plugin to get the supported operations +func (p *PluginManager) getSupportedOperations(tablePluginClient *grpc.PluginClient) (*proto.GetSupportedOperationsResponse, error) { + supportedOperations, err := tablePluginClient.GetSupportedOperations() + if err != nil { + // if the plugin does not implement GetSupportedOperations, it will return a NotImplemented error + // just return an empty response + if helpers.IsNotGRPCImplementedError(err) { + return &proto.GetSupportedOperationsResponse{}, nil + } + } + return supportedOperations, err +} + // Describe starts the plugin if needed, and returns the plugin description, including description of any custom formats -func (p *PluginManager) Describe(ctx context.Context, pluginName string, opts ...DescribeOpts) (*types.DescribeResponse, error) { +func (p *PluginManager) Describe(_ context.Context, pluginName string, opts ...DescribeOpts) (*types.DescribeResponse, error) { // build plugin ref from the name pluginDef := pplugin.NewPlugin(pluginName) @@ -205,7 +279,7 @@ func (p *PluginManager) UpdateCollectionState(ctx context.Context, partition *co _, err = pluginClient.UpdateCollectionState(req) if err != nil { - return fmt.Errorf("error updating collection state for plugin %s: %w", pluginClient.Name, error_helpers.TransformErrorToSteampipe(err)) + return fmt.Errorf("error updating collection state for plugin %s: %w", pluginClient.Name, error_helpers.TransformErrorToTailpipe(err)) } // just return - the observer is responsible for waiting for completion @@ -264,26 +338,26 @@ func (p *PluginManager) formatToProto(ctx context.Context, format *config.Format return &proto.FormatData{Name: format.FullName, Regex: desc.Regex}, nil } -func (p *PluginManager) getSourcePluginReattach(ctx context.Context, partition *config.Partition, tablePlugin *pplugin.Plugin) (*proto.SourcePluginReattach, error) { +func (p *PluginManager) getSourcePluginReattach(ctx context.Context, partition *config.Partition, tablePlugin *pplugin.Plugin) (*grpc.PluginClient, *proto.SourcePluginReattach, error) { // identify which plugin provides the source sourcePlugin, err := p.determineSourcePlugin(partition) if err != nil { - return nil, fmt.Errorf("error determining plugin for source %s: %w", partition.Source.Type, err) + return nil, nil, fmt.Errorf("error determining plugin for source %s: %w", partition.Source.Type, err) } // if this plugin is different from the plugin that provides the table, we need to start the source plugin, // and then pass reattach info if sourcePlugin.Plugin == tablePlugin.Plugin { - return nil, nil + return nil, nil, nil } // so the source plugin is different from the table plugin - start if needed sourcePluginClient, err := p.getPlugin(sourcePlugin) if err != nil { - return nil, fmt.Errorf("error starting plugin '%s' required for source '%s': %w", sourcePlugin.Alias, partition.Source.Type, err) + return nil, nil, fmt.Errorf("error starting plugin '%s' required for source '%s': %w", sourcePlugin.Alias, partition.Source.Type, err) } sourcePluginReattach := proto.NewSourcePluginReattach(partition.Source.Type, sourcePlugin.Alias, sourcePluginClient.Client.ReattachConfig()) - return sourcePluginReattach, nil + return sourcePluginClient, sourcePluginReattach, nil } // getExecutionId generates a unique id based on the current time @@ -320,21 +394,21 @@ func (p *PluginManager) getPlugin(pluginDef *pplugin.Plugin) (*grpc.PluginClient func (p *PluginManager) startPlugin(tp *pplugin.Plugin) (*grpc.PluginClient, error) { pluginName := tp.Alias - pluginPath, err := pfilepaths.GetPluginPath(tp.Plugin, tp.Alias) - if err != nil { - return nil, fmt.Errorf("error getting plugin path for plugin '%s': %w", tp.Alias, err) - } - // create the plugin map pluginMap := map[string]goplugin.Plugin{ pluginName: &shared.TailpipeGRPCPlugin{}, } + cmd, err := p.getPluginCommand(tp) + if err != nil { + return nil, err + } + pluginStartTimeout := p.getPluginStartTimeout() c := goplugin.NewClient(&goplugin.ClientConfig{ HandshakeConfig: shared.Handshake, Plugins: pluginMap, - Cmd: exec.Command("sh", "-c", pluginPath), + Cmd: cmd, AllowedProtocols: []goplugin.Protocol{goplugin.ProtocolGRPC}, // send plugin stderr (logging) to our stderr Stderr: os.Stderr, @@ -355,6 +429,24 @@ func (p *PluginManager) startPlugin(tp *pplugin.Plugin) (*grpc.PluginClient, err return client, nil } +func (p *PluginManager) getPluginCommand(tp *pplugin.Plugin) (*exec.Cmd, error) { + pluginPath, err := pfilepaths.GetPluginPath(tp.Plugin, tp.Alias) + if err != nil { + return nil, fmt.Errorf("error getting plugin path for plugin '%s': %w", tp.Alias, err) + } + + cmd := exec.Command("sh", "-c", pluginPath) + + // set the max memory for the plugin (if specified) + maxMemoryBytes := tp.GetMaxMemoryBytes() + if maxMemoryBytes != 0 { + slog.Info("Setting max memory for plugin", "plugin", tp.Alias, "max memory (mb)", maxMemoryBytes/(1024*1024)) + // set GOMEMLIMIT for the plugin command env + cmd.Env = append(os.Environ(), fmt.Sprintf("GOMEMLIMIT=%d", maxMemoryBytes)) + } + return cmd, nil +} + // for debug purposes, plugin start timeout can be set via an environment variable TAILPIPE_PLUGIN_START_TIMEOUT func (p *PluginManager) getPluginStartTimeout() time.Duration { pluginStartTimeout := 1 * time.Minute @@ -369,54 +461,69 @@ func (p *PluginManager) getPluginStartTimeout() time.Duration { return pluginStartTimeout } -func (p *PluginManager) readCollectionEvents(ctx context.Context, pluginStream proto.TailpipePlugin_AddObserverClient) { - pluginEventChan := make(chan *proto.Event) - errChan := make(chan error) +func (p *PluginManager) readCollectionEvents(ctx context.Context, executionId string, pluginStream proto.TailpipePlugin_AddObserverClient) { + pluginEventChan := make(chan events.Event) + slog.Info("starting to read plugin events") // goroutine to read the plugin event stream and send the events down the event channel go func() { defer func() { if r := recover(); r != nil { - errChan <- helpers.ToError(r) + pluginEventChan <- events.NewCompletedEvent(executionId, 0, 0, gokithelpers.ToError(r)) } + // ensure + close(pluginEventChan) }() for { - e, err := pluginStream.Recv() + pe, err := pluginStream.Recv() + if err != nil { + slog.Error("error reading from plugin stream", "error", err) + // clean up the error + err = cleanupPluginError(err) + // send a completion event with an error + pluginEventChan <- events.NewCompletedEvent(executionId, 0, 0, err) + return + } + e, err := events.EventFromProto(pe) if err != nil { - errChan <- err + slog.Info("error converting plugin event to proto", "error", err) + // send a completion event with an error + pluginEventChan <- events.NewCompletedEvent(executionId, 0, 0, err) return } + pluginEventChan <- e + // if this is a completion event , stop polling + if pe.GetCompleteEvent() != nil { + slog.Info("got completion event - stop polling and close plugin event channel") + return + } } }() - // loop until the context is cancelled + // loop until either: + // - the context is cancelled + // - there is an error reading from the plugin stream + // - the pluginEventChan is closed (following a completion event) for { select { case <-ctx.Done(): return - case err := <-errChan: - if err != nil { - // TODO #error handle error - // ignore EOF errors - if !strings.Contains(err.Error(), "EOF") { - fmt.Printf("Error reading from plugin stream: %s\n", err.Error()) //nolint:forbidigo// TODO #error - } - return - } - case protoEvent := <-pluginEventChan: + + case ev := <-pluginEventChan: // convert the protobuf event to an observer event // and send it to the observer - if protoEvent == nil { - // TODO #error unexpected - raise an error - send error to observers + if ev == nil { + // channel is closed return } - p.obs.Notify(protoEvent) - // TODO #error should we stop polling if we get an error event? - // if this is a completion event (or other error event???), stop polling - if protoEvent.GetCompleteEvent() != nil { - close(pluginEventChan) + err := p.obs.Notify(ctx, ev) + if err != nil { + // if notify fails, send a completion event with the error + if err = p.obs.Notify(ctx, events.NewCompletedEvent(executionId, 0, 0, err)); err != nil { + slog.Error("error notifying observer of error", "error", err) + } return } } @@ -424,7 +531,7 @@ func (p *PluginManager) readCollectionEvents(ctx context.Context, pluginStream p } -// determineSourcePlugin determines plugin which provides trhe given source type for the given partition +// determineSourcePlugin determines plugin which provides the given source type for the given partition // try to use the source information registered in the version file // if older plugins are installed which did not register the source type, then fall back to deducing the plugin name func (p *PluginManager) determineSourcePlugin(partition *config.Partition) (*pplugin.Plugin, error) { @@ -435,7 +542,10 @@ func (p *PluginManager) determineSourcePlugin(partition *config.Partition) (*ppl return nil, fmt.Errorf("error describing sources: %w", err) } if _, ok := coreSources[sourceType]; ok { - return pplugin.NewPlugin(constants.CorePluginName), nil + // Rather than hard code to core@latest, call CorePluginInstallStream + // to handle the case where the core plugin is not installed + coreName := constants.CorePluginInstallStream() + return pplugin.NewPlugin(coreName), nil } pluginName := config.GetPluginForSourceType(sourceType, config.GlobalConfig.PluginVersions) @@ -461,20 +571,25 @@ func EnsureCorePlugin(ctx context.Context) (*versionfile.PluginVersionFile, erro action := "Installing" // check if core plugin is already installed - exists, _ := pplugin.Exists(ctx, constants.CorePluginName) + corePluginRequiredConstraint := constants.CorePluginRequiredVersionConstraint() + corePluginStream := constants.CorePluginInstallStream() + exists, _ := pplugin.Exists(ctx, corePluginStream) if exists { // check if the min version is satisfied; if not then update // find the version of the core plugin from the pluginVersions - installedVersion := pluginVersions.Plugins[constants.CorePluginFullName].Version + // NOTE: use the prefixed name to index the pluginVersions map + fullName := constants.CorePluginFullName() + installedVersion := pluginVersions.Plugins[fullName].Version + // if installed version is 'local', that will do if installedVersion == "local" { return pluginVersions, nil } // compare the version(using semver) with the min version - satisfy, err := checkSatisfyMinVersion(installedVersion, constants.MinCorePluginVersion) + satisfy, err := versionSatisfyVersionConstraint(installedVersion, corePluginRequiredConstraint) if err != nil { return nil, err } @@ -487,7 +602,7 @@ func EnsureCorePlugin(ctx context.Context) (*versionfile.PluginVersionFile, erro action = "Updating" } // install the core plugin - if err = installCorePlugin(ctx, state, action); err != nil { + if err = installCorePlugin(ctx, state, action, corePluginStream); err != nil { return nil, err } @@ -503,7 +618,6 @@ func loadPluginVersionFile(ctx context.Context) (*versionfile.PluginVersionFile, return nil, err } - // TODO KAI CHECK THIS // add any "local" plugins (i.e. plugins installed under the 'local' folder) into the version file ew := pluginVersions.AddLocalPlugins(ctx) if ew.Error != nil { @@ -512,15 +626,16 @@ func loadPluginVersionFile(ctx context.Context) (*versionfile.PluginVersionFile, return pluginVersions, nil } -func installCorePlugin(ctx context.Context, state installationstate.InstallationState, operation string) error { +func installCorePlugin(ctx context.Context, state installationstate.InstallationState, operation string, pluginStream string) error { spinner := statushooks.NewStatusSpinnerHook() spinner.Show() defer spinner.Hide() spinner.SetStatus(fmt.Sprintf("%s core plugin", operation)) - // get the latest version of the core plugin - ref := pociinstaller.NewImageRef(constants.CorePluginName) + // get a ref for the plugin stream + ref := pociinstaller.NewImageRef(pluginStream) org, name, constraint := ref.GetOrgNameAndStream() + rpv, err := pplugin.GetLatestPluginVersionByConstraint(ctx, state.InstallationID, org, name, constraint) if err != nil { return err @@ -537,21 +652,26 @@ func installCorePlugin(ctx context.Context, state installationstate.Installation return nil } -func checkSatisfyMinVersion(ver string, pluginVersion string) (bool, error) { +func versionSatisfyVersionConstraint(ver string, pluginVersion string) (bool, error) { // check if the version satisfies the min version requirement of core plugin // Parse the versions installedVer, err := version.NewVersion(ver) if err != nil { return false, err } - minReq, err := version.NewVersion(pluginVersion) + versionConstraint, err := version.NewConstraint(pluginVersion) if err != nil { return false, err } - // compare the versions - if installedVer.LessThan(minReq) { - return false, nil + return versionConstraint.Check(installedVer), nil +} + +func IsNotImplementedError(err error) bool { + status, ok := status.FromError(err) + if !ok { + return false } - return true, nil + + return status.Code() == codes.Unimplemented } diff --git a/internal/query/execute.go b/internal/query/execute.go index 6857827a..653dd7cf 100644 --- a/internal/query/execute.go +++ b/internal/query/execute.go @@ -9,7 +9,6 @@ import ( "strings" "time" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/query" "github.com/turbot/pipe-fittings/v2/querydisplay" "github.com/turbot/pipe-fittings/v2/queryresult" @@ -17,6 +16,7 @@ import ( "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/tailpipe/internal/config" "github.com/turbot/tailpipe/internal/database" + error_helpers "github.com/turbot/tailpipe/internal/error_helpers" ) func RunBatchSession(ctx context.Context, args []string, db *database.DuckDb) (int, []error) { @@ -43,18 +43,26 @@ func RunBatchSession(ctx context.Context, args []string, db *database.DuckDb) (i } func ExecuteQuery(ctx context.Context, query string, db *database.DuckDb) (int, error) { + // Get column definitions first + colDefs, err := GetColumnDefsForQuery(query, db) + if err != nil { + // if this error is due to trying to select a table which exists in partition config, + // but there is no view defined (as no rows have been collected), return a special error + err := handleMissingViewError(err) + return 0, err + } + // Run the query rows, err := db.QueryContext(ctx, query) if err != nil { // if this error is due to trying to select a table which exists in partition config, // but there is no view defined (as no rows have been collected), return a special error err := handleMissingViewError(err) - return 0, err } // Execute the query - result, err := Execute(ctx, rows) + result, err := Execute(ctx, rows, colDefs) if err != nil { return 0, err } @@ -62,12 +70,58 @@ func ExecuteQuery(ctx context.Context, query string, db *database.DuckDb) (int, // show output _, rowErrors := querydisplay.ShowOutput(ctx, result) if rowErrors > 0 { - // TODO #errors find a way to return the error + // TODO #errors find a way to return the error https://github.com/turbot/pipe-fittings/issues/745 return rowErrors, fmt.Errorf("query execution failed") } return 0, nil } +// GetColumnDefsForQuery executes a DESCRIBE query to get column definitions +func GetColumnDefsForQuery(query string, db *database.DuckDb) ([]*queryresult.ColumnDef, error) { + // Remove trailing semicolon from query to avoid DESCRIBE syntax errors + cleanQuery := strings.TrimSpace(query) + cleanQuery = strings.TrimSuffix(cleanQuery, ";") + + // Create DESCRIBE query + describeQuery := fmt.Sprintf("DESCRIBE (%s)", cleanQuery) + + // Execute the describe query + rows, err := db.Query(describeQuery) + if err != nil { + return nil, err + } + defer rows.Close() + + // Initialize a slice to hold column definitions + var columnDefs []*queryresult.ColumnDef + + // Process the DESCRIBE results + for rows.Next() { + var columnName, columnType string + var nullable, key, defaultValue, extra sql.NullString + + // DESCRIBE returns: column_name, column_type, null, key, default, extra + err := rows.Scan(&columnName, &columnType, &nullable, &key, &defaultValue, &extra) + if err != nil { + return nil, err + } + + columnDef := &queryresult.ColumnDef{ + Name: columnName, + DataType: columnType, + OriginalName: columnName, + } + + columnDefs = append(columnDefs, columnDef) + } + + if err := rows.Err(); err != nil { + return nil, err + } + + return columnDefs, nil +} + func handleMissingViewError(err error) error { errorMessage := err.Error() // Define the regex to match the table name @@ -97,12 +151,7 @@ func (t TimingMetadata) GetTiming() any { return t } -func Execute(ctx context.Context, rows *sql.Rows) (res *queryresult.Result[TimingMetadata], err error) { - - colDefs, err := fetchColumnDefs(rows) - if err != nil { - return nil, err - } +func Execute(ctx context.Context, rows *sql.Rows, colDefs []*queryresult.ColumnDef) (res *queryresult.Result[TimingMetadata], err error) { result := queryresult.NewResult[TimingMetadata](colDefs, TimingMetadata{}) @@ -151,34 +200,3 @@ func streamResults(ctx context.Context, rows *sql.Rows, result *queryresult.Resu } statushooks.Done(ctx) } - -// FetchColumnDefs extracts column definitions from sql.Rows and returns a slice of ColumnDef. -func fetchColumnDefs(rows *sql.Rows) ([]*queryresult.ColumnDef, error) { - // Get column names - columnNames, err := rows.Columns() - if err != nil { - return nil, err - } - - // Get column types - columnTypes, err := rows.ColumnTypes() - if err != nil { - return nil, err - } - - // Initialize a slice to hold column definitions - var columnDefs []*queryresult.ColumnDef - - for i, colType := range columnTypes { - columnDef := &queryresult.ColumnDef{ - Name: columnNames[i], - DataType: colType.DatabaseTypeName(), - OriginalName: columnNames[i], // Set this if you have a way to obtain the original name (optional) - this would be needed when multiple same columns are requested - } - - // Append to the list of column definitions - columnDefs = append(columnDefs, columnDef) - } - - return columnDefs, nil -} diff --git a/internal/query/execute_test.go b/internal/query/execute_test.go new file mode 100644 index 00000000..3794a5e6 --- /dev/null +++ b/internal/query/execute_test.go @@ -0,0 +1,457 @@ +package query + +import ( + "fmt" + "github.com/turbot/pipe-fittings/v2/filepaths" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/turbot/tailpipe/internal/database" +) + +func TestGetColumnDefsForQuery(t *testing.T) { + filepaths.PipesInstallDir = "." + + // Create a temporary DuckDB instance for testing + db, err := database.NewDuckDb(database.WithTempDir(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + // Create test tables with sample data + setupTestTables(t, db) + + tests := []struct { + name string + query string + expectedCols []string + expectError bool + }{ + { + name: "simple select", + query: "SELECT id, name, value FROM test_table", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "select with aliases", + query: "SELECT id AS user_id, name AS user_name, value AS score FROM test_table", + expectedCols: []string{"user_id", "user_name", "score"}, + expectError: false, + }, + { + name: "select with functions", + query: "SELECT COUNT(*), AVG(value), MAX(name) FROM test_table", + expectedCols: []string{"count_star()", "avg(\"value\")", "max(\"name\")"}, + expectError: false, + }, + { + name: "inner join", + query: "SELECT t1.id, t1.name, t2.category FROM test_table t1 INNER JOIN category_table t2 ON t1.id = t2.id", + expectedCols: []string{"id", "name", "category"}, + expectError: false, + }, + { + name: "left join", + query: "SELECT t1.id, t1.name, t2.category FROM test_table t1 LEFT JOIN category_table t2 ON t1.id = t2.id", + expectedCols: []string{"id", "name", "category"}, + expectError: false, + }, + { + name: "right join", + query: "SELECT t1.id, t1.name, t2.category FROM test_table t1 RIGHT JOIN category_table t2 ON t1.id = t2.id", + expectedCols: []string{"id", "name", "category"}, + expectError: false, + }, + { + name: "full outer join", + query: "SELECT t1.id, t1.name, t2.category FROM test_table t1 FULL OUTER JOIN category_table t2 ON t1.id = t2.id", + expectedCols: []string{"id", "name", "category"}, + expectError: false, + }, + { + name: "cross join", + query: "SELECT t1.id, t1.name, t2.category FROM test_table t1 CROSS JOIN category_table t2", + expectedCols: []string{"id", "name", "category"}, + expectError: false, + }, + { + name: "group by", + query: "SELECT category, COUNT(*) as count, AVG(value) as avg_value FROM test_table t1 JOIN category_table t2 ON t1.id = t2.id GROUP BY category", + expectedCols: []string{"category", "count", "avg_value"}, + expectError: false, + }, + { + name: "group by with having", + query: "SELECT category, COUNT(*) as count FROM test_table t1 JOIN category_table t2 ON t1.id = t2.id GROUP BY category HAVING COUNT(*) > 1", + expectedCols: []string{"category", "count"}, + expectError: false, + }, + { + name: "order by", + query: "SELECT id, name, value FROM test_table ORDER BY value DESC", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "limit and offset", + query: "SELECT id, name, value FROM test_table ORDER BY id LIMIT 5 OFFSET 2", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "subquery in select", + query: "SELECT id, name, (SELECT AVG(value) FROM test_table) as avg_all FROM test_table", + expectedCols: []string{"id", "name", "avg_all"}, + expectError: false, + }, + { + name: "subquery in from", + query: "SELECT * FROM (SELECT id, name, value FROM test_table WHERE value > 5) as sub", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "subquery in where", + query: "SELECT id, name, value FROM test_table WHERE value > (SELECT AVG(value) FROM test_table)", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "union", + query: "SELECT id, name FROM test_table UNION SELECT id, category as name FROM category_table", + expectedCols: []string{"id", "name"}, + expectError: false, + }, + { + name: "union all", + query: "SELECT id, name FROM test_table UNION ALL SELECT id, category as name FROM category_table", + expectedCols: []string{"id", "name"}, + expectError: false, + }, + { + name: "case statement", + query: "SELECT id, name, CASE WHEN value > 5 THEN 'high' WHEN value > 2 THEN 'medium' ELSE 'low' END as level FROM test_table", + expectedCols: []string{"id", "name", "level"}, + expectError: false, + }, + { + name: "window function", + query: "SELECT id, name, value, ROW_NUMBER() OVER (ORDER BY value DESC) as rank FROM test_table", + expectedCols: []string{"id", "name", "value", "rank"}, + expectError: false, + }, + { + name: "window function with partition", + query: "SELECT id, name, value, ROW_NUMBER() OVER (PARTITION BY name ORDER BY value DESC) as rank FROM test_table", + expectedCols: []string{"id", "name", "value", "rank"}, + expectError: false, + }, + { + name: "aggregate with window function", + query: "SELECT id, name, value, AVG(value) OVER (PARTITION BY name) as avg_by_name FROM test_table", + expectedCols: []string{"id", "name", "value", "avg_by_name"}, + expectError: false, + }, + { + name: "cte (common table expression)", + query: "WITH cte AS (SELECT id, name, value FROM test_table WHERE value > 3) SELECT * FROM cte", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "multiple ctes", + query: "WITH cte1 AS (SELECT id, name FROM test_table), cte2 AS (SELECT id, category FROM category_table) SELECT cte1.name, cte2.category FROM cte1 JOIN cte2 ON cte1.id = cte2.id", + expectedCols: []string{"name", "category"}, + expectError: false, + }, + { + name: "exists subquery", + query: "SELECT id, name FROM test_table WHERE EXISTS (SELECT 1 FROM category_table WHERE category_table.id = test_table.id)", + expectedCols: []string{"id", "name"}, + expectError: false, + }, + { + name: "in subquery", + query: "SELECT id, name FROM test_table WHERE id IN (SELECT id FROM category_table WHERE category = 'A')", + expectedCols: []string{"id", "name"}, + expectError: false, + }, + { + name: "complex nested query", + query: "SELECT t1.id, t1.name, t2.category, (SELECT COUNT(*) FROM test_table WHERE value > t1.value) as higher_count FROM test_table t1 LEFT JOIN category_table t2 ON t1.id = t2.id WHERE t1.value > (SELECT AVG(value) FROM test_table) ORDER BY t1.value DESC LIMIT 10", + expectedCols: []string{"id", "name", "category", "higher_count"}, + expectError: false, + }, + { + name: "query with semicolon", + query: "SELECT id, name, value FROM test_table;", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "query with extra whitespace", + query: " SELECT id, name, value FROM test_table ", + expectedCols: []string{"id", "name", "value"}, + expectError: false, + }, + { + name: "invalid query", + query: "SELECT * FROM non_existent_table", + expectedCols: nil, + expectError: true, + }, + { + name: "syntax error", + query: "SELECT id, name FROM test_table WHERE", + expectedCols: nil, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + colDefs, err := GetColumnDefsForQuery(tt.query, db) + + if tt.expectError { + assert.Error(t, err) + assert.Nil(t, colDefs) + } else { + assert.NoError(t, err) + assert.NotNil(t, colDefs) + assert.Len(t, colDefs, len(tt.expectedCols)) + + // Verify column names match expected + for i, expectedCol := range tt.expectedCols { + if i < len(colDefs) { + assert.Equal(t, expectedCol, colDefs[i].Name, "Column name mismatch at position %d", i) + assert.Equal(t, expectedCol, colDefs[i].OriginalName, "Original name should match name for position %d", i) + } + } + + // Verify all column definitions have data types + for i, colDef := range colDefs { + assert.NotEmpty(t, colDef.DataType, "Column %d (%s) should have a data type", i, colDef.Name) + } + } + }) + } +} + +func TestGetColumnDefsForQuery_EdgeCases(t *testing.T) { + // Create a temporary DuckDB instance for testing + db, err := database.NewDuckDb(database.WithTempDir(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + // Create test tables with sample data + setupTestTables(t, db) + + tests := []struct { + name string + query string + description string + expectError bool + }{ + { + name: "empty query", + query: "", + description: "Empty query should return error", + expectError: true, + }, + { + name: "whitespace only query", + query: " \t\n ", + description: "Whitespace-only query should return error", + expectError: true, + }, + { + name: "query with comments", + query: "SELECT id, name -- comment\nFROM test_table /* another comment */", + description: "Query with comments should work", + expectError: false, + }, + { + name: "query with special characters in column names", + query: "SELECT id as \"user-id\", name as \"user_name\", value as \"score_value\" FROM test_table", + description: "Query with quoted column names should work", + expectError: false, + }, + { + name: "query with numeric column names", + query: "SELECT 1 as \"1\", 2 as \"2\", 3 as \"3\" FROM test_table", + description: "Query with numeric column names should work", + expectError: false, + }, + { + name: "query with very long column names", + query: "SELECT id as \"very_long_column_name_that_exceeds_normal_length_limits_for_testing_purposes\", name, value FROM test_table", + description: "Query with very long column names should work", + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + colDefs, err := GetColumnDefsForQuery(tt.query, db) + + if tt.expectError { + assert.Error(t, err, tt.description) + assert.Nil(t, colDefs) + } else { + assert.NoError(t, err, tt.description) + assert.NotNil(t, colDefs) + assert.Greater(t, len(colDefs), 0, "Should return at least one column definition") + } + }) + } +} + +func TestGetColumnDefsForQuery_DataTypes(t *testing.T) { + // Create a temporary DuckDB instance for testing + db, err := database.NewDuckDb(database.WithTempDir(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + // Create test table with various data types + _, err = db.Exec(` + CREATE TABLE data_types_test ( + id INTEGER, + name VARCHAR, + value DOUBLE, + is_active BOOLEAN, + created_at TIMESTAMP, + data BLOB, + json_data JSON, + uuid_val UUID + ) + `) + require.NoError(t, err) + + // Insert test data + _, err = db.Exec(` + INSERT INTO data_types_test VALUES + (1, 'test', 3.14, true, '2024-01-01 10:00:00', 'binary_data', '{"key": "value"}', '123e4567-e89b-12d3-a456-426614174000') + `) + require.NoError(t, err) + + tests := []struct { + name string + query string + expectedTypes []string + }{ + { + name: "all data types", + query: "SELECT * FROM data_types_test", + expectedTypes: []string{"INTEGER", "VARCHAR", "DOUBLE", "BOOLEAN", "TIMESTAMP", "BLOB", "JSON", "UUID"}, + }, + { + name: "type casting", + query: "SELECT CAST(id AS BIGINT) as big_id, CAST(value AS DECIMAL(10,2)) as decimal_value FROM data_types_test", + expectedTypes: []string{"BIGINT", "DECIMAL(10,2)"}, + }, + { + name: "string functions", + query: "SELECT UPPER(name) as upper_name, LENGTH(name) as name_length, SUBSTRING(name, 1, 2) as name_sub FROM data_types_test", + expectedTypes: []string{"VARCHAR", "BIGINT", "VARCHAR"}, + }, + { + name: "numeric functions", + query: "SELECT ABS(value) as abs_value, ROUND(value, 2) as rounded_value, CEIL(value) as ceil_value FROM data_types_test", + expectedTypes: []string{"DOUBLE", "DOUBLE", "DOUBLE"}, + }, + { + name: "date functions", + query: "SELECT YEAR(created_at) as year, MONTH(created_at) as month, DAY(created_at) as day FROM data_types_test", + expectedTypes: []string{"BIGINT", "BIGINT", "BIGINT"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + colDefs, err := GetColumnDefsForQuery(tt.query, db) + require.NoError(t, err) + require.Len(t, colDefs, len(tt.expectedTypes)) + + for i, expectedType := range tt.expectedTypes { + // DuckDB might return slightly different type names, so we check if the type contains our expected type + assert.Contains(t, colDefs[i].DataType, expectedType, + "Column %d (%s) should have data type containing %s, got %s", + i, colDefs[i].Name, expectedType, colDefs[i].DataType) + } + }) + } +} + +// setupTestTables creates test tables with sample data for testing +func setupTestTables(t testing.TB, db *database.DuckDb) { + // Create test_table + _, err := db.Exec(` + CREATE TABLE test_table ( + id INTEGER, + name VARCHAR, + value DOUBLE + ) + `) + require.NoError(t, err) + + // Create category_table + _, err = db.Exec(` + CREATE TABLE category_table ( + id INTEGER, + category VARCHAR + ) + `) + require.NoError(t, err) + + // Insert test data into test_table + _, err = db.Exec(` + INSERT INTO test_table VALUES + (1, 'Alice', 10.5), + (2, 'Bob', 7.2), + (3, 'Charlie', 15.8), + (4, 'David', 3.1), + (5, 'Eve', 12.3) + `) + require.NoError(t, err) + + // Insert test data into category_table + _, err = db.Exec(` + INSERT INTO category_table VALUES + (1, 'A'), + (2, 'B'), + (3, 'A'), + (4, 'C'), + (5, 'B') + `) + require.NoError(t, err) +} + +// BenchmarkGetColumnDefsForQuery benchmarks the GetColumnDefsForQuery function +func BenchmarkGetColumnDefsForQuery(b *testing.B) { + // Create a temporary DuckDB instance for benchmarking + db, err := database.NewDuckDb(database.WithTempDir(b.TempDir())) + require.NoError(b, err) + defer db.Close() + + // Setup test tables + setupTestTables(b, db) + + queries := []string{ + "SELECT id, name, value FROM test_table", + "SELECT t1.id, t1.name, t2.category FROM test_table t1 INNER JOIN category_table t2 ON t1.id = t2.id", + "SELECT category, COUNT(*) as count, AVG(value) as avg_value FROM test_table t1 JOIN category_table t2 ON t1.id = t2.id GROUP BY category", + "SELECT id, name, value, ROW_NUMBER() OVER (ORDER BY value DESC) as rank FROM test_table", + } + + for i, query := range queries { + b.Run(fmt.Sprintf("Query_%d", i+1), func(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := GetColumnDefsForQuery(query, db) + if err != nil { + b.Fatalf("GetColumnDefsForQuery failed: %v", err) + } + } + }) + } +} diff --git a/main.go b/main.go index f923bd1e..176917a0 100644 --- a/main.go +++ b/main.go @@ -7,11 +7,11 @@ import ( "github.com/spf13/viper" "github.com/turbot/go-kit/helpers" "github.com/turbot/pipe-fittings/v2/constants" - "github.com/turbot/pipe-fittings/v2/error_helpers" "github.com/turbot/pipe-fittings/v2/utils" "github.com/turbot/tailpipe/cmd" "github.com/turbot/tailpipe/internal/cmdconfig" localconstants "github.com/turbot/tailpipe/internal/constants" + "github.com/turbot/tailpipe/internal/error_helpers" ) var exitCode int diff --git a/scripts/linux_container_info.sh b/scripts/linux_container_info.sh new file mode 100755 index 00000000..0a16f884 --- /dev/null +++ b/scripts/linux_container_info.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# This is a script to get the information about the linux container. +# Used in release smoke tests. + +uname -a # uname information +cat /etc/os-release # OS version information +ldd --version # glibc version information \ No newline at end of file diff --git a/scripts/prepare_amazonlinux_container.sh b/scripts/prepare_amazonlinux_container.sh new file mode 100755 index 00000000..7ca92813 --- /dev/null +++ b/scripts/prepare_amazonlinux_container.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# This is a script to install dependencies/packages, create user, and assign necessary permissions in the Amazon Linux 2023 container. +# Used in release smoke tests. + +set -e # Exit on any error + +# update yum and install required packages +yum install -y shadow-utils tar gzip ca-certificates jq curl --allowerasing + +# Extract the tailpipe binary +tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin + +# Make the binary executable +chmod +x /usr/local/bin/tailpipe + +# Make the scripts executable +chmod +x /scripts/smoke_test.sh + +echo "Amazon Linux container preparation completed successfully" \ No newline at end of file diff --git a/scripts/prepare_centos_container.sh b/scripts/prepare_centos_container.sh new file mode 100755 index 00000000..4a07bbbb --- /dev/null +++ b/scripts/prepare_centos_container.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# This is a script to install dependencies/packages, create user, and assign necessary permissions in the CentOS Stream 9 container. +# Used in release smoke tests. + +set -e + +# update yum and install required packages +yum install -y epel-release tar ca-certificates jq curl --allowerasing + +# Extract the tailpipe binary +tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin + +# Make the binary executable +chmod +x /usr/local/bin/tailpipe + +# Make the scripts executable +chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/prepare_ubuntu_container.sh b/scripts/prepare_ubuntu_container.sh new file mode 100755 index 00000000..ee1a420c --- /dev/null +++ b/scripts/prepare_ubuntu_container.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# This is a script to install dependencies/packages, create user, and assign necessary permissions in the ubuntu 24 container. +# Used in release smoke tests. + +# update apt and install required packages +apt-get update +apt-get install -y tar ca-certificates jq gzip + +# Extract the tailpipe binary +tar -xzf /artifacts/linux.tar.gz -C /usr/local/bin + +# Make the binary executable +chmod +x /usr/local/bin/tailpipe + +# Make the scripts executable +chmod +x /scripts/smoke_test.sh \ No newline at end of file diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh new file mode 100755 index 00000000..424e0622 --- /dev/null +++ b/scripts/smoke_test.sh @@ -0,0 +1,112 @@ +#!/bin/sh +# This is a script with set of commands to smoke test a tailpipe build. +# The plan is to gradually add more tests to this script. + +set -e + +# Ensure the PATH includes the directory where jq is installed +export PATH=$PATH:/usr/local/bin:/usr/bin:/bin + +# Check jq is available +jq --version + +/usr/local/bin/tailpipe --version # check version + +# Test basic query functionality (should work without data) +/usr/local/bin/tailpipe query "SELECT 1 as smoke_test" # verify basic query works + +# Test connect functionality +DB_FILE=$(/usr/local/bin/tailpipe connect --output json | jq -r '.init_script_path') + +# Verify the database file exists +if [ -f "$DB_FILE" ]; then + echo "Database file exists" +else + echo "Database file not found: $DB_FILE" + exit 1 +fi + +# Test plugin installation +/usr/local/bin/tailpipe plugin install chaos # install chaos plugin for testing +/usr/local/bin/tailpipe plugin list # verify plugin is installed + +# Show available tables and sources after plugin installation +/usr/local/bin/tailpipe table list # should now show chaos tables +/usr/local/bin/tailpipe source list # should now show chaos sources + +# Create configuration for testing +# the config path is different for darwin and linux +if [ "$(uname -s)" = "Darwin" ]; then + CONFIG_DIR="$HOME/.tailpipe/config" +else + CONFIG_DIR="$HOME/.tailpipe/config" +fi + +mkdir -p "$CONFIG_DIR" + +# Create chaos.tpc configuration file +cat > "$CONFIG_DIR/chaos.tpc" << 'EOF' +partition "chaos_date_time" "chaos_date_time_range" { + source "chaos_date_time" { + row_count = 100 + } +} +EOF + +cat "$CONFIG_DIR/chaos.tpc" + +# Test partition listing after adding configuration +/usr/local/bin/tailpipe partition list # should now show the chaos partition + +# Show partition details +/usr/local/bin/tailpipe partition show chaos_date_time.chaos_date_time_range + +# Test data collection - this is the main goal! +# The chaos plugin generates dates around 2006-2007, so we need to collect from that range +echo "Starting data collection..." +# Use different timeout commands for macOS vs Linux +if [ "$(uname -s)" = "Darwin" ]; then + # macOS - try gtimeout first, fallback to no timeout + if command -v gtimeout >/dev/null 2>&1; then + gtimeout 300 /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false || { + echo "Collection timed out or failed, trying without timeout..." + /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false + } + else + echo "No timeout command available on macOS, running without timeout..." + /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false + fi +else + # Linux - use timeout + timeout 300 /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false || { + echo "Collection timed out or failed, trying without progress bar..." + /usr/local/bin/tailpipe collect chaos_date_time.chaos_date_time_range --from="2006-01-01" --progress=false 2>&1 | head -50 + echo "Collection attempt completed" + } +fi + +# Verify data was collected before proceeding +echo "Checking if data was collected..." +DATA_COUNT=$(/usr/local/bin/tailpipe query "SELECT COUNT(*) as count FROM chaos_date_time" --output json 2>/dev/null | jq -r '.rows[0].count' || echo "0") +echo "Data count: $DATA_COUNT" + +if [ "$DATA_COUNT" -gt 0 ]; then + echo "Data collection successful, proceeding with queries..." + + # Test querying collected data + # Query 1: Count total rows + /usr/local/bin/tailpipe query "SELECT COUNT(*) as total_rows FROM chaos_date_time" --output json + + # Query 2: Show first 5 rows + /usr/local/bin/tailpipe query "SELECT * FROM chaos_date_time LIMIT 5" --output table + + # Query 3: Basic aggregation using the correct column name + /usr/local/bin/tailpipe query "SELECT date_part('hour', timestamp) as hour, COUNT(*) as count FROM chaos_date_time GROUP BY date_part('hour', timestamp) ORDER BY hour LIMIT 5" --output json +else + echo "No data collected, skipping query tests..." + echo "Available tables after collection attempt:" + /usr/local/bin/tailpipe table list +fi + +# Test plugin show functionality +/usr/local/bin/tailpipe plugin show chaos diff --git a/tests/acceptance/run-local.sh b/tests/acceptance/run-local.sh index 6d3f9845..dc3ae902 100755 --- a/tests/acceptance/run-local.sh +++ b/tests/acceptance/run-local.sh @@ -24,8 +24,8 @@ set -e echo "Installation complete at $TAILPIPE_INSTALL_DIR" # install chaos plugin -tailpipe plugin install chaos -echo "Installed CHAOS plugin" +tailpipe plugin install chaos aws +echo "Installed CHAOS and AWS plugins" if [ $# -eq 0 ]; then # Run all test files diff --git a/tests/acceptance/run.sh b/tests/acceptance/run.sh index 5b5ffe9b..4dad7a11 100755 --- a/tests/acceptance/run.sh +++ b/tests/acceptance/run.sh @@ -10,6 +10,7 @@ export BATS_PATH=$MY_PATH/lib/bats/bin/bats export LIB_BATS_ASSERT=$MY_PATH/lib/bats-assert export LIB_BATS_SUPPORT=$MY_PATH/lib/bats-support export TEST_DATA_DIR=$MY_PATH/test_data/templates +export SOURCE_FILES_DIR=$MY_PATH/test_data/source_files # Must have these commands for the test suite to run declare -a required_commands=("sed" "tailpipe" $BATS_PATH "rm" "mv" "cp" "mkdir" "cd" "node" "npm" "npx" "head" "wc" "find" "basename" "dirname") @@ -52,5 +53,12 @@ if [ $# -eq 0 ]; then # Run all test files $BATS_PATH --tap $MY_PATH/test_files else - $BATS_PATH --tap $MY_PATH/test_files/${1} + # Handle each argument + for arg in "$@"; do + # If the path is relative, make it absolute relative to the test files directory + if [[ "$arg" != /* ]]; then + arg="$MY_PATH/test_files/$arg" + fi + $BATS_PATH --tap "$arg" + done fi diff --git a/tests/acceptance/test_data/source_files/aws_cloudtrail_flaws/flaws_cloudtrail00.json.gz b/tests/acceptance/test_data/source_files/aws_cloudtrail_flaws/flaws_cloudtrail00.json.gz new file mode 100644 index 00000000..afc133a9 Binary files /dev/null and b/tests/acceptance/test_data/source_files/aws_cloudtrail_flaws/flaws_cloudtrail00.json.gz differ diff --git a/tests/acceptance/test_data/source_files/aws_cloudtrail_flaws/flaws_cloudtrail01.json.gz b/tests/acceptance/test_data/source_files/aws_cloudtrail_flaws/flaws_cloudtrail01.json.gz new file mode 100644 index 00000000..69570d1e Binary files /dev/null and b/tests/acceptance/test_data/source_files/aws_cloudtrail_flaws/flaws_cloudtrail01.json.gz differ diff --git a/tests/acceptance/test_data/source_files/config_tests/workspace_tests.json b/tests/acceptance/test_data/source_files/config_tests/workspace_tests.json new file mode 100644 index 00000000..f03b0759 --- /dev/null +++ b/tests/acceptance/test_data/source_files/config_tests/workspace_tests.json @@ -0,0 +1,155 @@ +[ { + "test": "env variables set, no command line arguments set and no workspace env variable set", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_UPDATE_CHECK=false", + "TAILPIPE_MEMORY_MAX_MB=16384", + "TAILPIPE_PLUGIN_MEMORY_MAX_MB=2048", + "TAILPIPE_TEMP_DIR_MAX_MB=8192" + ], + "args": [] + }, + "expected": { + "update-check": "false", + "memory-max-mb": 16384, + "memory-max-mb-plugin": 2048, + "temp-dir-max-mb": 8192 + } + }, + { + "test": "only command line arguments set and no env variables set", + "description": "", + "cmd": "query", + "setup": { + "env": [], + "args": [ + "--workspace=workspace_profiles" + ] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + }, + { + "test": "workspace env variable set and no command line arguments set", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_WORKSPACE=workspace_profiles" + ], + "args": [] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + }, + { + "test": "All env variables set and command line argument set", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_WORKSPACE=development", + "TAILPIPE_LOG_LEVEL=debug", + "TAILPIPE_UPDATE_CHECK=false", + "TAILPIPE_MEMORY_MAX_MB=16384", + "TAILPIPE_MEMORY_MAX_MB_PLUGIN=2048", + "TAILPIPE_TEMP_DIR_MAX_MB=8192" + ], + "args": [ + "--workspace=workspace_profiles" + ] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + }, + { + "test": "workspace env variable set and --workspace arg passed", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_WORKSPACE=development" + ], + "args": [ + "--workspace=workspace_profiles" + ] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + }, + { + "test": "all env variables set including workspace env variable and no command line arguments set", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_WORKSPACE=development", + "TAILPIPE_LOG_LEVEL=debug", + "TAILPIPE_UPDATE_CHECK=false", + "TAILPIPE_MEMORY_MAX_MB=16384", + "TAILPIPE_MEMORY_MAX_MB_PLUGIN=2048", + "TAILPIPE_TEMP_DIR_MAX_MB=8192" + ], + "args": [] + }, + "expected": { + "workspace": "development", + "log-level": "warn", + "update-check": "false", + "memory-max-mb": 512, + "memory-max-mb-plugin": 206, + "temp-dir-max-mb": 512 + } + }, + { + "test": "all env variables set except workspace env variable and --workspace arg passed", + "description": "", + "cmd": "query", + "setup": { + "env": [ + "TAILPIPE_LOG_LEVEL=debug", + "TAILPIPE_UPDATE_CHECK=false", + "TAILPIPE_MEMORY_MAX_MB=16384", + "TAILPIPE_MEMORY_MAX_MB_PLUGIN=2048", + "TAILPIPE_TEMP_DIR_MAX_MB=8192" + ], + "args": [ + "--workspace=workspace_profiles" + ] + }, + "expected": { + "workspace": "workspace_profiles", + "log-level": "trace", + "update-check": "false", + "memory-max-mb": 1024, + "memory-max-mb-plugin": 512, + "temp-dir-max-mb": 1024 + } + } +] \ No newline at end of file diff --git a/tests/acceptance/test_data/source_files/config_tests/workspaces.tpc b/tests/acceptance/test_data/source_files/config_tests/workspaces.tpc new file mode 100755 index 00000000..d38c3592 --- /dev/null +++ b/tests/acceptance/test_data/source_files/config_tests/workspaces.tpc @@ -0,0 +1,16 @@ + +workspace "workspace_profiles" { + log_level = "trace" + update_check = false + memory_max_mb = 1024 + plugin_memory_max_mb = 512 + temp_dir_max_mb = 1024 +} + +workspace "development"{ + log_level = "warn" + update_check = false + memory_max_mb = 512 + plugin_memory_max_mb = 206 + temp_dir_max_mb = 512 +} \ No newline at end of file diff --git a/tests/acceptance/test_data/source_files/custom_logs/access_log.csv b/tests/acceptance/test_data/source_files/custom_logs/access_log.csv new file mode 100644 index 00000000..bd48357b --- /dev/null +++ b/tests/acceptance/test_data/source_files/custom_logs/access_log.csv @@ -0,0 +1,4 @@ +timestamp,ip_address,user_agent,status_code +2024-05-01T10:30:45Z,192.168.1.1,"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",200 +2024-05-01T10:31:00Z,192.168.1.2,"Mozilla/5.0 (Windows NT 10.0; Win64; x64)",404 +2024-05-01T10:32:15Z,192.168.1.3,"Mozilla/5.0 (Linux; Android 10)",200 \ No newline at end of file diff --git a/tests/acceptance/test_data/source_files/custom_logs/nested_patterns.log b/tests/acceptance/test_data/source_files/custom_logs/nested_patterns.log new file mode 100644 index 00000000..d5ab24cd --- /dev/null +++ b/tests/acceptance/test_data/source_files/custom_logs/nested_patterns.log @@ -0,0 +1,3 @@ +2024-05-01T10:30:45Z [INFO] [AWS] RequestID: req-1234, Service: s3, Operation: ListBuckets, Duration: 150ms +2024-05-01T10:31:00Z [ERROR] [AWS] RequestID: req-5678, Service: ec2, Operation: DescribeInstances, Error: {"code": "InvalidInstanceID", "message": "The instance ID 'i-1234567890abcdef0' does not exist"} +2024-05-01T10:32:15Z [DEBUG] [AWS] RequestID: req-9012, Service: lambda, Operation: Invoke, Duration: 45ms \ No newline at end of file diff --git a/tests/acceptance/test_data/source_files/custom_logs/null_if_data.csv b/tests/acceptance/test_data/source_files/custom_logs/null_if_data.csv new file mode 100644 index 00000000..979b7116 --- /dev/null +++ b/tests/acceptance/test_data/source_files/custom_logs/null_if_data.csv @@ -0,0 +1,6 @@ +id,status,value,description +1,active,42,normal value +2,inactive,0,zero value +3,active,-1,negative value +4,active,2,empty value +5,active,999,special value \ No newline at end of file diff --git a/tests/acceptance/test_data/source_files/custom_logs/plugin-2025-05-01.log b/tests/acceptance/test_data/source_files/custom_logs/plugin-2025-05-01.log new file mode 100644 index 00000000..4b852b35 --- /dev/null +++ b/tests/acceptance/test_data/source_files/custom_logs/plugin-2025-05-01.log @@ -0,0 +1,13 @@ +2025-04-28 15:16:35.733 UTC [DEBUG] steampipe-plugin-aws.plugin: [DEBUG] 1744125262935: retrying request Lambda/ListFunctions, attempt 8 +2025-04-28 15:16:35.733 UTC [INFO] steampipe-plugin-aws.plugin: [INFO] 1744125273258: BackoffDelay: attempt=8, retryTime=2m55.50675s, err=https response error StatusCode: 0, RequestID: , request send failed, Get "https://lambda.ap-northeast-1.amazonaws.com/2015-03-31/functions?MaxItems=10000": lookup lambda.ap-northeast-1.amazonaws.com on 192.168.1.254:53: read udp 192.168.1.204:57677->192.168.1.254:53: i/o timeout +2025-04-28 15:16:36.033 UTC [DEBUG] steampipe-plugin-aws.plugin: [DEBUG] 1744125262935: retrying request Lambda/ListFunctions, attempt 8 +2025-04-28 15:16:36.033 UTC [INFO] steampipe-plugin-aws.plugin: [INFO] 1744125273258: BackoffDelay: attempt=8, retryTime=2m16.14075s, err=https response error StatusCode: 0, RequestID: , request send failed, Get "https://lambda.ap-southeast-2.amazonaws.com/2015-03-31/functions?MaxItems=10000": lookup lambda.ap-southeast-2.amazonaws.com on 192.168.1.254:53: read udp 192.168.1.204:54718->192.168.1.254:53: i/o timeout +2025-04-28 15:16:38.463 UTC [DEBUG] steampipe-plugin-aws.plugin: [DEBUG] 1744125262935: retrying request Lambda/ListFunctions, attempt 8 +2025-04-28 15:16:38.464 UTC [INFO] steampipe-plugin-aws.plugin: [INFO] 1744125273258: BackoffDelay: attempt=8, retryTime=2m44.025s, err=https response error StatusCode: 0, RequestID: , request send failed, Get "https://lambda.ap-northeast-3.amazonaws.com/2015-03-31/functions?MaxItems=10000": lookup lambda.ap-northeast-3.amazonaws.com on 192.168.1.254:53: read udp 192.168.1.204:55845->192.168.1.254:53: i/o timeout +2025-04-28 15:16:58.442 UTC [INFO] PluginManager Shutdown +2025-04-28 15:16:58.442 UTC [INFO] PluginManager closing pool +2025-04-28 15:16:58.442 UTC [INFO] Kill plugin hub.steampipe.io/plugins/turbot/terraform@latest (0xc00092e100) +2025-04-28 15:16:58.442 UTC [DEBUG] PluginManager killPlugin start +2025-04-28 15:16:58.442 UTC [INFO] PluginManager killing plugin hub.steampipe.io/plugins/turbot/terraform@latest (81771) +2025-04-28 15:16:58.456 UTC [DEBUG] stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = error reading from server: EOF" +2025-04-28 15:16:58.460 UTC [INFO] plugin process exited: plugin=/Users/jsmyth/.steampipe/plugins/hub.steampipe.io/plugins/turbot/terraform@latest/steampipe-plugin-terraform.plugin id=81771 \ No newline at end of file diff --git a/tests/acceptance/test_data/source_files/custom_logs/server_metrics.jsonl b/tests/acceptance/test_data/source_files/custom_logs/server_metrics.jsonl new file mode 100644 index 00000000..435cd974 --- /dev/null +++ b/tests/acceptance/test_data/source_files/custom_logs/server_metrics.jsonl @@ -0,0 +1,3 @@ +{"timestamp": "2024-05-01T10:30:45Z", "server_id": "srv-001", "cpu_usage": 75.5, "memory_used": 8192, "is_healthy": true} +{"timestamp": "2024-05-01T10:31:00Z", "server_id": "srv-002", "cpu_usage": 90.2, "memory_used": 16384, "is_healthy": false} +{"timestamp": "2024-05-01T10:32:15Z", "server_id": "srv-003", "cpu_usage": 45.8, "memory_used": 4096, "is_healthy": true} \ No newline at end of file diff --git a/tests/acceptance/test_data/source_files/custom_logs/transform_data.csv b/tests/acceptance/test_data/source_files/custom_logs/transform_data.csv new file mode 100644 index 00000000..7c81a567 --- /dev/null +++ b/tests/acceptance/test_data/source_files/custom_logs/transform_data.csv @@ -0,0 +1,4 @@ +timestamp,raw_value,status_code,user_agent,ip_address,custom_time +2024-05-01T10:00:00Z,42,200,Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36,192.168.1.1,2024-05-01 10:00:00 +2024-05-01T10:01:00Z,99,404,Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7),10.0.0.1,2024-05-01 10:01:00 +2024-05-01T10:02:00Z,150,500,curl/7.68.0,172.16.0.1,2024-05-01 10:02:00 \ No newline at end of file diff --git a/tests/acceptance/test_files/config_precedence.bats b/tests/acceptance/test_files/config_precedence.bats new file mode 100644 index 00000000..32c1e28e --- /dev/null +++ b/tests/acceptance/test_files/config_precedence.bats @@ -0,0 +1,121 @@ +load "$LIB_BATS_ASSERT/load.bash" +load "$LIB_BATS_SUPPORT/load.bash" + +## workspace tests + +@test "generic config precedence test" { + cp $SOURCE_FILES_DIR/config_tests/workspaces.tpc $TAILPIPE_INSTALL_DIR/config/workspaces.tpc + # setup test folder and read the test-cases file + cd $SOURCE_FILES_DIR/config_tests + tests=$(cat workspace_tests.json) + # echo $tests + + # to create the failure message + err="" + flag=0 + + # fetch the keys(test names) + test_keys=$(echo $tests | jq '. | keys[]') + # echo $test_keys + + for i in $test_keys; do + # each test case do the following + unset TAILPIPE_INSTALL_DIR + cwd=$(pwd) + export TAILPIPE_CONFIG_DUMP=config_json + + # command accordingly + cmd=$(echo $tests | jq -c ".[${i}]" | jq ".cmd") + if [[ $cmd == '"query"' ]]; then + tp_cmd='tailpipe query "select 1"' + fi + # echo $tp_cmd + + # key=$(echo $i) + echo -e "\n" + test_name=$(echo $tests | jq -c ".[${i}]" | jq ".test") + echo ">>> TEST NAME: $test_name" + + # env variables needed for setup + env=$(echo $tests | jq -c ".[${i}]" | jq ".setup.env") + # echo $env + + # set env variables + for e in $(echo "${env}" | jq -r '.[]'); do + export $e + done + + # args to run with tailpipe query command + args=$(echo $tests | jq -c ".[${i}]" | jq ".setup.args") + echo $args + + # construct the tailpipe command to be run with the args + for arg in $(echo "${args}" | jq -r '.[]'); do + tp_cmd="${tp_cmd} ${arg}" + done + echo "tailpipe command: $tp_cmd" # help debugging in case of failures + + # get the actual config by running the constructed tailpipe command + run $tp_cmd + echo "output from tailpipe command: $output" # help debugging in case of failures + + # The output contains log lines followed by a JSON object + # Find the start of the JSON (line starting with '{') and extract from there to the end + # Then use jq to parse and compact it + json_start_line=$(echo "$output" | grep -n '^{' | tail -1 | cut -d: -f1) + if [[ -n "$json_start_line" ]]; then + config_json=$(echo "$output" | tail -n +$json_start_line) + else + # Fallback: try to find any JSON-like content + config_json=$(echo "$output" | grep -A 1000 '{' | head -1000) + fi + + # Parse with jq and handle errors gracefully + actual_config=$(echo "$config_json" | jq -c '.' 2>/dev/null) + if [[ $? -ne 0 ]] || [[ -z "$actual_config" ]]; then + echo "Failed to parse JSON config, raw output:" + echo "$config_json" + actual_config="{}" + fi + echo "actual config: \n$actual_config" # help debugging in case of failures + + # get expected config from test case + expected_config=$(echo $tests | jq -c ".[${i}]" | jq ".expected") + # echo $expected_config + + # fetch only keys from expected config + exp_keys=$(echo $expected_config | jq '. | keys[]' | jq -s 'flatten | @sh' | tr -d '\'\' | tr -d '"') + + for key in $exp_keys; do + # get the expected and the actual value for the keys + exp_val=$(echo $(echo $expected_config | jq --arg KEY $key '.[$KEY]' | tr -d '"')) + act_val=$(echo $(echo $actual_config | jq --arg KEY $key '.[$KEY]' | tr -d '"')) + + # get the absolute paths for install-dir and mod-location + if [[ $key == "install-dir" ]] || [[ $key == "mod-location" ]]; then + exp_val="${cwd}/${exp_val}" + fi + echo "expected $key: $exp_val" + echo "actual $key: $act_val" + + # check the values + if [[ "$exp_val" != "$act_val" ]]; then + flag=1 + err="FAILED: $test_name >> key: $key ; expected: $exp_val ; actual: $act_val \n${err}" + fi + done + + # check if all passed + if [[ $flag -eq 0 ]]; then + echo "PASSED āœ…" + else + echo "FAILED āŒ" + fi + # reset flag back to 0 for the next test case + flag=0 + done + echo -e "\n" + echo -e "$err" + assert_equal "$err" "" + rm -f err +} diff --git a/tests/acceptance/test_files/core_formats.bats b/tests/acceptance/test_files/core_formats.bats new file mode 100644 index 00000000..218a4af4 --- /dev/null +++ b/tests/acceptance/test_files/core_formats.bats @@ -0,0 +1,266 @@ +load "$LIB_BATS_ASSERT/load.bash" +load "$LIB_BATS_SUPPORT/load.bash" + +@test "verify grok format definition" { + cat << EOF > $TAILPIPE_INSTALL_DIR/config/format_grok.tpc +format "grok" "steampipe_plugin" { + layout = \`%{TIMESTAMP_ISO8601:timestamp} %{WORD:timezone} \[%{LOGLEVEL:severity}\]\s+(?:%{NOTSPACE:plugin_name}: \[%{LOGLEVEL:plugin_severity}\]\s+%{NUMBER:plugin_timestamp}:\s+)?%{GREEDYDATA:message}\` +} + +table "steampipe_plugin" { + format = format.grok.steampipe_plugin + + column "tp_timestamp" { + source = "timestamp" + } + + column "plugin_timestamp" { + type = "timestamp" + } +} + +partition "steampipe_plugin" "local" { + source "file" { + format = format.grok.steampipe_plugin + paths = ["$SOURCE_FILES_DIR/custom_logs/"] + file_layout = \`plugin-%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day}.log\` + } +} +EOF + + # Run collection and verify + tailpipe collect steampipe_plugin --progress=false --from=2025-04-26 + + # Verify data was collected correctly + run tailpipe query "select plugin_name, tp_timestamp from steampipe_plugin limit 1" --output csv + echo $output + + assert_equal "$output" "plugin_name,tp_timestamp +steampipe-plugin-aws.plugin,2025-04-28 15:16:35" + + # Cleanup + rm -rf $TAILPIPE_INSTALL_DIR/config/format_grok.tpc +} + +@test "verify delimited format definition" { + cat << EOF > $TAILPIPE_INSTALL_DIR/config/format_delimited.tpc +format "delimited" "access_log" { + delimiter = "," + header = true +} + +table "access_log" { + format = format.delimited.access_log + + column "tp_timestamp" { + source = "timestamp" + } + + column "ip_address" { + type = "varchar" + } + + column "user_agent" { + type = "varchar" + } + + column "status_code" { + type = "integer" + } +} + +partition "access_log" "local" { + source "file" { + format = format.delimited.access_log + paths = ["$SOURCE_FILES_DIR/custom_logs/"] + file_layout = "access_log.csv" + } +} +EOF + + # Run collection and verify + tailpipe collect access_log --progress=false --from=2024-04-30 + + # Verify data was collected correctly + run tailpipe query "select ip_address, status_code from access_log limit 1" --output csv + echo $output + + assert_equal "$output" "ip_address,status_code +192.168.1.1,200" + + # Cleanup + rm -rf $TAILPIPE_INSTALL_DIR/config/format_delimited.tpc +} + +@test "verify jsonl format definition" { + cat << EOF > $TAILPIPE_INSTALL_DIR/config/format_jsonl.tpc +format "jsonl" "server_metrics" { + description = "Server metrics in JSON Lines format" +} + +table "server_metrics" { + format = format.jsonl.server_metrics + + column "tp_timestamp" { + source = "timestamp" + } + + column "server_id" { + type = "varchar" + } + + column "cpu_usage" { + type = "float" + } + + column "memory_used" { + type = "integer" + } + + column "is_healthy" { + type = "boolean" + } +} + +partition "server_metrics" "local" { + source "file" { + format = format.jsonl.server_metrics + paths = ["$SOURCE_FILES_DIR/custom_logs/"] + file_layout = "server_metrics.jsonl" + } +} +EOF + + # Run collection and verify + tailpipe collect server_metrics --progress=false --from=2024-04-30 + + # Verify data was collected correctly + run tailpipe query "select server_id, cpu_usage, memory_used, is_healthy from server_metrics limit 1" --output csv + echo $output + + assert_equal "$output" "server_id,cpu_usage,memory_used,is_healthy +srv-001,75.5,8192,true" + + # Cleanup + rm -rf $TAILPIPE_INSTALL_DIR/config/format_jsonl.tpc +} + +@test "verify regex format definition" { + cat << EOF > $TAILPIPE_INSTALL_DIR/config/format_regex.tpc +format "regex" "plugin_log" { + layout = \`^(?P\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}\.\d{3})\s+(?P\w+)\s+\[(?P\w+)\]\s+(?P[\w.-]+)?(?:\s*:\s*\[(?P\w+)\]\s+(?P\d+):\s+)?(?P.*)\` +} + +table "plugin_log" { + format = format.regex.plugin_log + + column "tp_timestamp" { + source = "timestamp" + } + + column "log_level" { + type = "varchar" + } + + column "plugin_name" { + type = "varchar" + } + + column "plugin_log_level" { + type = "varchar" + } + + column "message" { + type = "varchar" + } +} + +partition "plugin_log" "local" { + source "file" { + format = format.regex.plugin_log + paths = ["$SOURCE_FILES_DIR/custom_logs/"] + file_layout = \`plugin-%{YEAR:year}-%{MONTHNUM:month}-%{MONTHDAY:day}.log\` + } +} +EOF + + # Run collection and verify + tailpipe collect plugin_log --progress=false --from=2025-04-28 + + # Verify data was collected correctly + run tailpipe query "select log_level, plugin_name, message from plugin_log limit 1" --output csv + echo $output + + assert_equal "$output" "log_level,plugin_name,message +DEBUG,steampipe-plugin-aws.plugin,\"retrying request Lambda/ListFunctions, attempt 8\"" + + # Cleanup + rm -rf $TAILPIPE_INSTALL_DIR/config/format_regex.tpc +} + +@test "verify grok format with nested patterns" { + cat << EOF > $TAILPIPE_INSTALL_DIR/config/format_grok_nested.tpc +format "grok" "aws_log" { + layout = \`%{TIMESTAMP_ISO8601:timestamp} \[%{LOGLEVEL:log_level}\] \[AWS\] RequestID: %{NOTSPACE:request_id}, Service: %{WORD:service}, Operation: %{WORD:operation}(?:, Duration: %{NUMBER:duration}ms)?(?:, Error: %{GREEDYDATA:error})?\` +} + +table "aws_log" { + format = format.grok.aws_log + + column "tp_timestamp" { + source = "timestamp" + } + + column "log_level" { + type = "varchar" + } + + column "request_id" { + type = "varchar" + } + + column "service" { + type = "varchar" + } + + column "operation" { + type = "varchar" + } + + column "duration" { + type = "integer" + } + + column "error" { + type = "varchar" + } +} + +partition "aws_log" "local" { + source "file" { + format = format.grok.aws_log + paths = ["$SOURCE_FILES_DIR/custom_logs/"] + file_layout = "nested_patterns.log" + } +} +EOF + + # Run collection and verify + tailpipe collect aws_log --progress=false --from=2024-04-30 + + # Verify data was collected correctly + run tailpipe query "select log_level, service, operation, duration from aws_log order by request_id" --output csv + echo $output + + assert_equal "$output" "log_level,service,operation,duration +INFO,s3,ListBuckets,150 +ERROR,ec2,DescribeInstances, +DEBUG,lambda,Invoke,45" + + # Cleanup + rm -rf $TAILPIPE_INSTALL_DIR/config/format_grok_nested.tpc +} + +function teardown() { + rm -rf $TAILPIPE_INSTALL_DIR/data +} \ No newline at end of file diff --git a/tests/acceptance/test_files/file_source.bats b/tests/acceptance/test_files/file_source.bats new file mode 100644 index 00000000..735d56cf --- /dev/null +++ b/tests/acceptance/test_files/file_source.bats @@ -0,0 +1,125 @@ +load "$LIB_BATS_ASSERT/load.bash" +load "$LIB_BATS_SUPPORT/load.bash" + +@test "verify file source logs count" { + cat << EOF > $TAILPIPE_INSTALL_DIR/config/cloudtrail_logs.tpc +partition "aws_cloudtrail_log" "fs" { + source "file" { + file_layout = ".json.gz" + paths = ["$SOURCE_FILES_DIR/aws_cloudtrail_flaws/"] + } +} +EOF + + cat $TAILPIPE_INSTALL_DIR/config/cloudtrail_logs.tpc + + # tailpipe collect + tailpipe collect aws_cloudtrail_log.fs --progress=false --from 2014-01-01 + + # run tailpipe query and verify the row counts + run tailpipe query "select count(*) as count from aws_cloudtrail_log;" --output csv + echo $output + + # We expect at least some records from the two files + assert_output --regexp 'count +200000' + + # remove the config file + rm -f $TAILPIPE_INSTALL_DIR/config/cloudtrail_logs.tpc +} + +@test "verify file source with multiple paths" { + # Create a second directory with the same files for testing multiple paths + mkdir -p $SOURCE_FILES_DIR/aws_cloudtrail_flaws2/ + cp $SOURCE_FILES_DIR/aws_cloudtrail_flaws/* $SOURCE_FILES_DIR/aws_cloudtrail_flaws2/ + + cat << EOF > $TAILPIPE_INSTALL_DIR/config/multi_path.tpc +partition "aws_cloudtrail_log" "fs2" { + source "file" { + file_layout = ".json.gz" + paths = ["$SOURCE_FILES_DIR/aws_cloudtrail_flaws/", "$SOURCE_FILES_DIR/aws_cloudtrail_flaws2/"] + } +} +EOF + + cat $TAILPIPE_INSTALL_DIR/config/multi_path.tpc + ls -al $SOURCE_FILES_DIR/aws_cloudtrail_flaws2 + + tailpipe plugin list + + # tailpipe collect + tailpipe collect aws_cloudtrail_log.fs2 --progress=false --from 2014-01-01 + + # run tailpipe query and verify the row counts + run tailpipe query "select count(*) as count from aws_cloudtrail_log;" --output csv + echo $output + + # We expect double the records since we're reading from two identical directories + assert_output --regexp 'count +400000' + + # remove the config file and test directory + rm -f $TAILPIPE_INSTALL_DIR/config/multi_path.tpc + rm -rf $SOURCE_FILES_DIR/aws_cloudtrail_flaws2/ +} + +@test "verify file source with custom file layout" { + cat << EOF > $TAILPIPE_INSTALL_DIR/config/custom_layout.tpc +partition "aws_cloudtrail_log" "fs3" { + source "file" { + paths = ["$SOURCE_FILES_DIR/aws_cloudtrail_flaws/"] + file_layout = \`flaws_cloudtrail%{NUMBER:file_number}.json.gz\` + } +} +EOF + + cat $TAILPIPE_INSTALL_DIR/config/custom_layout.tpc + + # tailpipe collect + tailpipe collect aws_cloudtrail_log.fs3 --progress=false --from 2014-01-01 + + # run tailpipe query and verify the row counts + run tailpipe query "select count(*) as count from aws_cloudtrail_log;" --output csv + echo $output + + # We expect the same number of records as the basic test + assert_output --regexp 'count +200000' + + # remove the config file + rm -f $TAILPIPE_INSTALL_DIR/config/custom_layout.tpc +} + +@test "verify file source with custom patterns" { + cat << EOF > $TAILPIPE_INSTALL_DIR/config/custom_patterns.tpc +partition "aws_cloudtrail_log" "fs4" { + source "file" { + paths = ["$SOURCE_FILES_DIR/aws_cloudtrail_flaws/"] + file_layout = \`%{MY_PATTERN}.json.gz\` + patterns = { + "MY_PATTERN": \`flaws_cloudtrail%{NUMBER:file_number}\` + } + } +} +EOF + + cat $TAILPIPE_INSTALL_DIR/config/custom_patterns.tpc + + # tailpipe collect + tailpipe collect aws_cloudtrail_log.fs4 --progress=false --from 2014-01-01 + + # run tailpipe query and verify the row counts + run tailpipe query "select count(*) as count from aws_cloudtrail_log;" --output csv + echo $output + + # We expect the same number of records as the basic test + assert_output --regexp 'count +200000' + + # remove the config file + rm -f $TAILPIPE_INSTALL_DIR/config/custom_patterns.tpc +} + +function teardown() { + rm -rf $TAILPIPE_INSTALL_DIR/data +} \ No newline at end of file diff --git a/tests/acceptance/test_files/from_and_to.bats b/tests/acceptance/test_files/from_and_to.bats index b44e4fff..71db1874 100644 --- a/tests/acceptance/test_files/from_and_to.bats +++ b/tests/acceptance/test_files/from_and_to.bats @@ -2,6 +2,7 @@ load "$LIB_BATS_ASSERT/load.bash" load "$LIB_BATS_SUPPORT/load.bash" @test "verify --from works in tailpipe query" { + skip "TODO - re-enable this test, when this feature is implemented in ducklake - https://github.com/turbot/tailpipe/issues/543" cat << EOF > $TAILPIPE_INSTALL_DIR/config/chaos_date_time.tpc partition "chaos_date_time" "date_time_inc" { source "chaos_date_time" { @@ -28,6 +29,7 @@ EOF } @test "verify --from works when ISO 8601 datetime is passed" { + skip "TODO - re-enable this test, when this feature is implemented in ducklake - https://github.com/turbot/tailpipe/issues/543" cat << EOF > $TAILPIPE_INSTALL_DIR/config/chaos_date_time.tpc partition "chaos_date_time" "date_time_inc" { source "chaos_date_time" { @@ -54,6 +56,7 @@ EOF } @test "verify --from works when ISO 8601 datetime with milliseconds is passed" { + skip "TODO - re-enable this test, when this feature is implemented in ducklake - https://github.com/turbot/tailpipe/issues/543" cat << EOF > $TAILPIPE_INSTALL_DIR/config/chaos_date_time.tpc partition "chaos_date_time" "date_time_inc" { source "chaos_date_time" { @@ -80,6 +83,7 @@ EOF } @test "verify --from works when RFC 3339 datetime with timezone is passed" { + skip "TODO - re-enable this test, when this feature is implemented in ducklake - https://github.com/turbot/tailpipe/issues/543" cat << EOF > $TAILPIPE_INSTALL_DIR/config/chaos_date_time.tpc partition "chaos_date_time" "date_time_inc" { source "chaos_date_time" { diff --git a/tests/acceptance/test_files/partition_delete.bats b/tests/acceptance/test_files/partition_delete.bats new file mode 100644 index 00000000..117e82d3 --- /dev/null +++ b/tests/acceptance/test_files/partition_delete.bats @@ -0,0 +1,59 @@ +load "$LIB_BATS_ASSERT/load.bash" +load "$LIB_BATS_SUPPORT/load.bash" + +@test "verify partition deletion" { + # Create a test partition configuration + cat << EOF > $TAILPIPE_INSTALL_DIR/config/delete_test.tpc +partition "chaos_all_columns" "delete_test" { + source "chaos_all_columns" { + row_count = 20 + } +} +EOF + + # Run tailpipe collect to create the partition + run tailpipe collect chaos_all_columns.delete_test --progress=false + echo $output + + # Verify data has been collected by checking partition list + run tailpipe partition list --output json + echo $output + + # Use jq to check partition exists with correct row count + local row_count=$(echo "$output" | jq '.[] | select(.name == "chaos_all_columns.delete_test") | .local.row_count') + assert_equal "$row_count" "20" + + # Run partition delete + run tailpipe partition delete chaos_all_columns.delete_test --force + echo $output + + # Verify the partition was deleted by checking partition list + run tailpipe partition list --output json + echo $output + + # Use jq to verify partition has no data + local file_count=$(echo "$output" | jq '.[] | select(.name == "chaos_all_columns.delete_test") | .local.file_count') + local file_size=$(echo "$output" | jq '.[] | select(.name == "chaos_all_columns.delete_test") | .local.file_size') + assert_equal "$file_count" "0" + assert_equal "$file_size" "0" + + # Clean up config file + rm -rf $TAILPIPE_INSTALL_DIR/config/delete_test.tpc +} + +@test "verify deletion of non-existent partition fails gracefully" { + # Attempt to delete a partition that doesn't exist + run tailpipe partition delete chaos_all_columns.non_existent_partition --force + echo $output + + # Verify the command failed with appropriate error message + assert_failure + assert_output --partial "partition not found" + + # Verify the error message is user-friendly and clear + assert_output --partial "chaos_all_columns.non_existent_partition" +} + +function teardown() { + rm -rf $TAILPIPE_INSTALL_DIR/data +} \ No newline at end of file diff --git a/tests/acceptance/test_files/partition_tests.bats b/tests/acceptance/test_files/partition_tests.bats new file mode 100644 index 00000000..1272e1fb --- /dev/null +++ b/tests/acceptance/test_files/partition_tests.bats @@ -0,0 +1,261 @@ +load "$LIB_BATS_ASSERT/load.bash" +load "$LIB_BATS_SUPPORT/load.bash" + +@test "verify partition with filter" { + # Create a test partition configuration with filter + cat << EOF > $TAILPIPE_INSTALL_DIR/config/filter_test.tpc +partition "chaos_all_columns" "filter_test_1" { + filter = "id % 2 = 0" + source "chaos_all_columns" { + row_count = 10 + } +} +EOF + + # Run tailpipe collect + tailpipe collect chaos_all_columns.filter_test_1 --progress=false + + # Run tailpipe query and verify the filtered data + run tailpipe query "select count(*) as count from chaos_all_columns" --output csv + echo $output + + # Based on actual output - should be 5 rows (half of 10) + assert_equal "$output" "count +5" + + # Clean up config file + rm -rf $TAILPIPE_INSTALL_DIR/config/filter_test.tpc +} + +@test "verify duplicate partition names" { + # Create a test partition configuration with duplicate partition names + cat << EOF > $TAILPIPE_INSTALL_DIR/config/duplicate_test.tpc +partition "chaos_all_columns" "duplicate_test_1" { + source "chaos_all_columns" { + row_count = 5 + } +} + +partition "chaos_all_columns" "duplicate_test_1" { + source "chaos_all_columns" { + row_count = 10 + } +} +EOF + + # Run tailpipe collect and check for error message + run tailpipe collect chaos_all_columns.duplicate_test_1 --progress=false + echo $output + + # Verify that the output contains the specific error message about duplicate partition + assert_output --partial "partition duplicate_test_1 already exists for table chaos_all_columns" + + # Clean up config file + rm -rf $TAILPIPE_INSTALL_DIR/config/duplicate_test.tpc +} + +@test "verify invalid filter syntax" { + skip "TODO - re-enable this test, when the error handling is fixed in ducklake - https://github.com/turbot/tailpipe/issues/544" + # Create a test partition configuration with invalid filter + cat << EOF > $TAILPIPE_INSTALL_DIR/config/invalid_filter_test.tpc +partition "chaos_all_columns" "invalid_filter_test_1" { + filter = "invalid_column = 1" # This column doesn't exist + source "chaos_all_columns" { + row_count = 5 + } +} +EOF + + # Run tailpipe collect and check for error message + run tailpipe collect chaos_all_columns.invalid_filter_test_1 --progress=false + echo $output + + # Verify that the output contains the specific error message about the invalid filter + assert_output --partial "Binder Error: Referenced column \"invalid_column\" not found in FROM clause!" + assert_output --partial "Errors: 5" # Verify that there were errors in processing + + # Clean up config file + rm -rf $TAILPIPE_INSTALL_DIR/config/invalid_filter_test.tpc +} + +@test "verify non-existent source reference" { + # Create a test partition configuration with non-existent source + cat << EOF > $TAILPIPE_INSTALL_DIR/config/invalid_source_test.tpc +partition "chaos_all_columns" "invalid_source_test" { + source "non_existent_source" { + row_count = 5 + } +} +EOF + + # Run tailpipe collect and check for error message + run tailpipe collect chaos_all_columns.invalid_source_test --progress=false + echo $output + + # Verify that the output contains the specific error message about the non-existent source + assert_output --partial "error starting plugin 'non' required for source 'non_existent_source'" + assert_output --partial "no plugin installed matching 'non'" + + # Clean up config file + rm -rf $TAILPIPE_INSTALL_DIR/config/invalid_source_test.tpc +} + +@test "verify missing source block from partition" { + skip "Re-enable after fixing the issue with missing source block" + # Create a test partition configuration without a source block + cat << EOF > $TAILPIPE_INSTALL_DIR/config/missing_source_test.tpc +partition "chaos_all_columns" "missing_source_test" { + # Intentionally missing source block +} +EOF + + # Run tailpipe collect and check for error message + run tailpipe collect chaos_all_columns.missing_source_test --progress=false + echo $output + + # Verify that the output contains the specific error message about missing source + assert_output --partial "Partition chaos_all_columns.missing_source_test is missing required source block" + assert_output --partial "A source block is required for every partition to specify the data source" + + # Clean up config file + rm -rf $TAILPIPE_INSTALL_DIR/config/missing_source_test.tpc +} + +@test "verify partition with non-existent table name" { + # Create a test partition configuration with a non-existent table name + cat << EOF > $TAILPIPE_INSTALL_DIR/config/invalid_table_test.tpc +partition "non_existent_table" "test_partition" { + source "chaos_all_columns" { + row_count = 10 + } +} +EOF + + # Run tailpipe collect and check for error message + run tailpipe collect non_existent_table.test_partition --progress=false + echo $output + + # Verify that the output contains the specific error message about invalid table + assert_output --partial "error starting plugin non" + assert_output --partial "no plugin installed matching 'non'" + + # Clean up config file + rm -rf $TAILPIPE_INSTALL_DIR/config/invalid_table_test.tpc +} + +@test "verify partition with invalid table name format" { + # Create a test partition configuration with an invalid table name format + cat << EOF > $TAILPIPE_INSTALL_DIR/config/invalid_format_test.tpc +partition "invalid.table.name" "test_partition" { + source "chaos_all_columns" { + row_count = 10 + } +} +EOF + + # Run tailpipe collect and check for error message + run tailpipe collect invalid.table.name.test_partition --progress=false + echo $output + + # Verify that the output contains the specific error message about invalid table name format + assert_output --partial "Invalid name: A name must start with a letter or underscore and may contain only letters, digits, underscores, and dashes" + + # Clean up config file + rm -rf $TAILPIPE_INSTALL_DIR/config/invalid_format_test.tpc +} + +@test "verify incompatible source type for table" { + # Create a test partition configuration using an incompatible source type + cat << EOF > $TAILPIPE_INSTALL_DIR/config/incompatible_source_test.tpc +partition "chaos_date_time" "incompatible_source_test" { + source "chaos_all_columns" { + row_count = 10 + } +} +EOF + + # Run tailpipe collect and check for error message + run tailpipe collect chaos_date_time.incompatible_source_test --progress=false + echo $output + + # Verify that the output contains the specific error message about incompatible source + assert_output --partial "source type chaos_all_columns not supported by table chaos_date_time" + + # Clean up config file + rm -rf $TAILPIPE_INSTALL_DIR/config/incompatible_source_test.tpc +} + +@test "verify behavior when no partitions match pattern" { + # Create a test partition configuration with a specific name + cat << EOF > $TAILPIPE_INSTALL_DIR/config/no_match_test.tpc +partition "chaos_all_columns" "specific_partition" { + source "chaos_all_columns" { + row_count = 5 + } +} +EOF + + # Run tailpipe collect with a pattern that won't match any partitions + run tailpipe collect chaos_all_columns.non_matching_* --progress=false + echo $output + + # Verify that the output contains the correct error message + assert_output --partial "Error: failed to get partition config: partition not found: chaos_all_columns.non_matching_*" + + # Verify that no data was collected + run tailpipe query "select count(*) as count from chaos_all_columns" --output csv + echo $output + + # Should show warning that no data has been collected + assert_output --partial "Warning: query 1 of 1 failed: no data has been collected for table chaos_all_columns" + + # Clean up config file + rm -rf $TAILPIPE_INSTALL_DIR/config/no_match_test.tpc +} + +@test "verify multiple matching partitions are collected correctly" { + # Create multiple test partition configurations + cat << EOF > $TAILPIPE_INSTALL_DIR/config/wildcard_test.tpc +partition "chaos_all_columns" "wildcard_test_1" { + source "chaos_all_columns" { + row_count = 5 + } +} + +partition "chaos_all_columns" "wildcard_test_2" { + source "chaos_all_columns" { + row_count = 5 + } +} + +partition "chaos_all_columns" "wildcard_test_3" { + source "chaos_all_columns" { + row_count = 5 + } +} +EOF + + # Run tailpipe collect with wildcard pattern + run tailpipe collect chaos_all_columns.wildcard_test_* --progress=false + echo $output + + # Verify that all partitions were collected successfully + assert_output --partial "Collecting logs for chaos_all_columns.wildcard_test_1" + assert_output --partial "Collecting logs for chaos_all_columns.wildcard_test_2" + assert_output --partial "Collecting logs for chaos_all_columns.wildcard_test_3" + + # Verify the total row count across all partitions + run tailpipe query "select count(*) as count from chaos_all_columns" --output csv + echo $output + + # Should be 15 rows total (5 rows per partition) + assert_equal "$output" "count +15" + + # Clean up config file + rm -rf $TAILPIPE_INSTALL_DIR/config/wildcard_test.tpc +} + +function teardown() { + rm -rf $TAILPIPE_INSTALL_DIR/data +} \ No newline at end of file diff --git a/tests/acceptance/test_files/plugin.bats b/tests/acceptance/test_files/plugin.bats new file mode 100644 index 00000000..419efd0a --- /dev/null +++ b/tests/acceptance/test_files/plugin.bats @@ -0,0 +1,59 @@ +load "$LIB_BATS_ASSERT/load.bash" +load "$LIB_BATS_SUPPORT/load.bash" + +@test "verify metadata in versions.json file after plugin install" { + # Ensure chaos plugin is installed (it should already be in acceptance tests) + run tailpipe plugin list --output json + echo $output + + # Verify chaos plugin is in the list + assert_output --partial "hub.tailpipe.io/plugins/turbot/chaos@latest" + + # Read the versions.json file + versions_file="$TAILPIPE_INSTALL_DIR/plugins/versions.json" + + # Verify the file exists + [ -f "$versions_file" ] + + # Read the file content + versions_content=$(cat "$versions_file") + echo "Versions file content: $versions_content" + + # Extract metadata for chaos plugin using jq + chaos_plugin_key="hub.tailpipe.io/plugins/turbot/chaos@latest" + + # Verify that metadata exists for the chaos plugin + metadata_exists=$(echo "$versions_content" | jq -r --arg key "$chaos_plugin_key" '.plugins | has($key) and (.[$key] | has("metadata"))') + assert_equal "$metadata_exists" "true" + + # Verify tables metadata - chaos plugin should have specific tables + tables=$(echo "$versions_content" | jq -r --arg key "$chaos_plugin_key" '.plugins[$key].metadata.tables // [] | sort | join(",")') + assert_equal "$tables" "chaos_all_columns,chaos_date_time,chaos_struct_columns" + + # Verify sources metadata - chaos plugin should have specific sources + sources=$(echo "$versions_content" | jq -r --arg key "$chaos_plugin_key" '.plugins[$key].metadata.sources // [] | sort | join(",")') + assert_equal "$sources" "chaos_all_columns,chaos_date_time,chaos_struct_columns" +} + +@test "verify format types and presets metadata exists in versions.json file after plugin install" { + # Read the versions.json file + versions_file="$TAILPIPE_INSTALL_DIR/plugins/versions.json" + + # Verify the file exists + [ -f "$versions_file" ] + + # Read the file content + versions_content=$(cat "$versions_file") + echo "Versions file content: $versions_content" + + # Test format_types and format_presets from core plugin (which has them) + core_plugin_key="hub.tailpipe.io/plugins/turbot/core@latest" + + # Verify format_types content - should contain the expected types + format_types=$(echo "$versions_content" | jq -r --arg key "$core_plugin_key" '.plugins[$key].metadata.format_types // [] | sort | join(",")') + assert_equal "$format_types" "delimited,grok,jsonl,regex" + + # Verify format_presets content - should contain the expected presets + format_presets=$(echo "$versions_content" | jq -r --arg key "$core_plugin_key" '.plugins[$key].metadata.format_presets // [] | sort | join(",")') + assert_equal "$format_presets" "delimited.default,jsonl.default" +} \ No newline at end of file diff --git a/tests/acceptance/test_files/table_block.bats b/tests/acceptance/test_files/table_block.bats new file mode 100644 index 00000000..baa8159a --- /dev/null +++ b/tests/acceptance/test_files/table_block.bats @@ -0,0 +1,153 @@ +load "$LIB_BATS_ASSERT/load.bash" +load "$LIB_BATS_SUPPORT/load.bash" + +@test "verify column transforms" { + cat << EOF > $TAILPIPE_INSTALL_DIR/config/format_delimited.tpc +format "delimited" "transform_test" { + delimiter = "," +} +EOF + + cat << EOF > $TAILPIPE_INSTALL_DIR/config/table_transform.tpc +table "transform_test" { + format = format.delimited.transform_test + + column "tp_timestamp" { + source = "timestamp" + type = "timestamp" + } + + column "tp_date" { + source = "timestamp" + type = "date" + } + + column "value_doubled" { + type = "integer" + transform = "raw_value * 2" + } + + column "status_category" { + type = "varchar" + transform = "CASE WHEN status_code < 300 THEN 'success' WHEN status_code < 400 THEN 'redirect' WHEN status_code < 500 THEN 'client_error' ELSE 'server_error' END" + } + + column "browser" { + type = "varchar" + transform = "CASE WHEN user_agent LIKE '%Windows%' THEN 'Windows' WHEN user_agent LIKE '%Macintosh%' THEN 'Mac' ELSE 'Other' END" + } + + column "is_internal" { + type = "boolean" + transform = "ip_address LIKE '192.168.%' OR ip_address LIKE '10.%' OR ip_address LIKE '172.16.%'" + } + + column "parsed_time" { + type = "timestamp" + transform = "strptime(CAST(custom_time AS VARCHAR), '%Y-%m-%d %H:%M:%S')" + } +} + +partition "transform_test" "local" { + source "file" { + format = format.delimited.transform_test + paths = ["$SOURCE_FILES_DIR/custom_logs/"] + file_layout = "transform_data.csv" + } +} +EOF + + # Run collection and verify + tailpipe collect transform_test --progress=false --from=2024-05-01 + + # Verify transformations + run tailpipe query "select value_doubled, status_category, browser, is_internal, parsed_time from transform_test order by tp_timestamp" --output csv + echo $output + + assert_equal "$output" "value_doubled,status_category,browser,is_internal,parsed_time +84,success,Windows,true,2024-05-01 10:00:00 +198,client_error,Mac,true,2024-05-01 10:01:00 +300,server_error,Other,true,2024-05-01 10:02:00" + + # Cleanup + rm -rf $TAILPIPE_INSTALL_DIR/config/format_delimited.tpc + rm -rf $TAILPIPE_INSTALL_DIR/config/table_transform.tpc +} + +@test "verify null_if in column blocks" { + # TODO - enable this test when null_if bug is fixed - https://github.com/turbot/tailpipe/issues/393 + skip "Enable this test when null_if bug is fixed" + cat << EOF > $TAILPIPE_INSTALL_DIR/config/format_delimited_null.tpc +format "delimited" "null_if_test" { + delimiter = "," +} +EOF + + cat << EOF > $TAILPIPE_INSTALL_DIR/config/table_null_if.tpc +table "null_if_test" { + format = format.delimited.null_if_test + + column "tp_timestamp" { + type = "timestamp" + transform = "now()" + } + + column "tp_date" { + type = "date" + transform = "current_date" + } + + column "id" { + type = "integer" + source = "id" + } + + column "status" { + type = "varchar" + source = "status" + null_if = "inactive" + } + + column "value" { + type = "integer" + source = "value" + null_if = "0" + } + + column "description" { + type = "varchar" + source = "description" + } +} + +partition "null_if_test" "local" { + source "file" { + format = format.delimited.null_if_test + paths = ["$SOURCE_FILES_DIR/custom_logs/"] + file_layout = "null_if_data.csv" + } +} +EOF + + # Run collection and verify + tailpipe collect null_if_test --progress=false --from=2024-05-01 + + # Verify null_if transformations + run tailpipe query "select id, status, value, description from null_if_test order by id" --output csv + echo $output + + assert_equal "$output" "id,status,value,description +1,active,42,normal value +2,,,zero value +3,active,-1,negative value +4,active,2,empty value +5,active,999,special value" + + # Cleanup + rm -rf $TAILPIPE_INSTALL_DIR/config/format_delimited_null.tpc + rm -rf $TAILPIPE_INSTALL_DIR/config/table_null_if.tpc +} + +function teardown() { + rm -rf $TAILPIPE_INSTALL_DIR/data +} \ No newline at end of file