turbot · pskrbasu · Jan 16, 2025 · Jan 10, 2025 · Jan 10, 2025 · Jan 13, 2025
diff --git a/.github/workflows/11-test-acceptance.yaml b/.github/workflows/11-test-acceptance.yaml
@@ -29,7 +29,7 @@ jobs:
         with:
           repository: turbot/pipe-fittings
           path: pipe-fittings
-          ref: tp
+          ref: tp-pragma
 
       - name: Checkout Tailpipe plugin SDK repository
         uses: actions/checkout@v4
@@ -82,121 +82,90 @@ jobs:
           if-no-files-found: error
 
   # TODO - Update the acceptance test job with the correct test suite
-  # acceptance_test:
-  #   name: Test
-  #   needs: goreleaser
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       platform: [ubuntu-latest]
-  #       test_block:
-  #         - "check"
-  #         - "resource_show_outputs"
-  #         - "dashboard"
-  #         - "backend"
-  #         - "mod"
-  #         - "mod_install"
-  #         - "sp_files"
-  #         - "var_resolution"
-  #         - "params_and_args"
-  #         - "snapshot"
-  #         - "dashboard_parsing_validation"
-  #         - "database_precedence"
-  #   runs-on: ${{ matrix.platform }}
-  #   steps:
-  #     - name: Checkout
-  #       uses: actions/checkout@v4
-  #       with:
-  #         submodules: true
-  #         path: powerpipe
-  #         ref: ${{ github.event.ref }}
-
-  #     - uses: actions/setup-go@v5
-  #       with:
-  #         go-version: 1.22
-  #         cache: false
-
-  #     - name: Prepare for downloads
-  #       id: prepare-for-downloads
-  #       run: |
-  #         mkdir ~/artifacts
+  acceptance_test:
+    name: Test
+    needs: goreleaser
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-latest]
+        test_block:
+          - "all_column_types"
+          - "from_and_to"
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          path: tailpipe
+          ref: ${{ github.event.ref }}
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: 1.22
+          cache: false
+
+      - name: Prepare for downloads
+        id: prepare-for-downloads
+        run: |
+          mkdir ~/artifacts
 
-  #     - name: Download Linux Build Artifacts
-  #       uses: actions/download-artifact@v3
-  #       if: ${{ matrix.platform == 'ubuntu-latest' }}
-  #       with:
-  #         name: build-artifact-linux
-  #         path: ~/artifacts
-
-  #     - name: Download Darwin Build Artifacts
-  #       uses: actions/download-artifact@v3
-  #       if: ${{ matrix.platform == 'macos-latest' }}
-  #       with:
-  #         name: build-artifact-darwin
-  #         path: ~/artifacts
-
-  #     - name: Extract Darwin Artifacts and Install Binary
-  #       if: ${{ matrix.platform == 'macos-latest' }}
-  #       run: |
-  #         mkdir ~/build
-  #         tar -xf ~/artifacts/darwin.tar.gz -C ~/build
-
-  #     - name: Extract Ubuntu Artifacts and Install Binary
-  #       if: ${{ matrix.platform == 'ubuntu-latest' }}
-  #       run: |
-  #         mkdir ~/build
-  #         tar -xf ~/artifacts/linux.tar.gz -C ~/build
-
-  #     - name: Install Steampipe(Darwin)
-  #       if: ${{ matrix.platform == 'macos-latest' }}
-  #       run: |
-  #         brew install turbot/tap/steampipe
-  #         steampipe -v
-
-  #     - name: Install Steampipe(Linux)
-  #       if: ${{ matrix.platform == 'ubuntu-latest' }}
-  #       run: |
-  #         sudo /bin/sh -c "$(curl -fsSL https://steampipe.io/install/steampipe.sh)"
-  #         steampipe -v
+      - name: Download Linux Build Artifacts
+        uses: actions/download-artifact@v3
+        if: ${{ matrix.platform == 'ubuntu-latest' }}
+        with:
+          name: build-artifact-linux
+          path: ~/artifacts
+
+      - name: Extract Ubuntu Artifacts and Install Binary
+        if: ${{ matrix.platform == 'ubuntu-latest' }}
+        run: |
+          mkdir ~/build
+          tar -xf ~/artifacts/linux.tar.gz -C ~/build
 
-  #     - name: Set PATH
-  #       run: |
-  #         echo "PATH=$PATH:$HOME/build:$GTIHUB_WORKSPACE/powerpipe/tests/acceptance/lib/bats-core/libexec" >> $GITHUB_ENV
-
-  #     - name: Go install jd
-  #       run: |
-  #         go install github.com/josephburnett/jd@latest
-
-  #     - name: Start steamipipe service
-  #       run: |
-  #         steampipe service start
-
-  #     - name: Run Test Suite
-  #       id: run-test-suite
-  #       timeout-minutes: 15
-  #       continue-on-error: true
-  #       run: |
-  #         chmod +x $GITHUB_WORKSPACE/powerpipe/tests/acceptance/run.sh
-  #         $GITHUB_WORKSPACE/powerpipe/tests/acceptance/run.sh ${{ matrix.test_block }}.bats
-  #         echo "exit_code=$(echo $?)" >> $GITHUB_OUTPUT
-  #         echo ">> here"
-
-  #     # This job checks whether the test suite has passed or not.
-  #     # Since the exit_code is set only when the bats test suite pass,
-  #     # we have added the if-conditional block
-  #     - name: Check Test Passed/Failed
-  #       if: ${{ success() }}
-  #       continue-on-error: false
-  #       run: |
-  #         if [ ${{ steps.run-test-suite.outputs.exit_code }} -eq 0 ]; then
-  #           exit 0
-  #         else
-  #           exit 1
-  #         fi
-
-  #     - name: Stop steampipe service
-  #       run: |
-  #         steampipe service stop
+      - name: Set PATH
+        run: |
+          echo "PATH=$PATH:$HOME/build:$GTIHUB_WORKSPACE/tailpipe/tests/acceptance/lib/bats-core/libexec" >> $GITHUB_ENV
+
+      - name: Go install jd
+        run: |
+          go install github.com/josephburnett/jd@latest
+
+      # TODO remove this step once tailpipe and its plugins are public
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.GH_USERNAME }}
+          password: ${{ secrets.GH_PAT }}
+
+      - name: Install Tailpipe and plugins
+        run: |
+          tailpipe plugin install chaos
+
+      - name: Run Test Suite
+        id: run-test-suite
+        timeout-minutes: 15
+        continue-on-error: true
+        run: |
+          chmod +x $GITHUB_WORKSPACE/tailpipe/tests/acceptance/run.sh
+          $GITHUB_WORKSPACE/tailpipe/tests/acceptance/run.sh ${{ matrix.test_block }}.bats
+          echo "exit_code=$(echo $?)" >> $GITHUB_OUTPUT
+          echo ">> here"
+
+      # This job checks whether the test suite has passed or not.
+      # Since the exit_code is set only when the bats test suite pass,
+      # we have added the if-conditional block
+      - name: Check Test Passed/Failed
+        if: ${{ success() }}
+        continue-on-error: false
+        run: |
+          if [ ${{ steps.run-test-suite.outputs.exit_code }} -eq 0 ]; then
+            exit 0
+          else
+            exit 1
+          fi
 
   cleanup:
     # let's clean up the artifacts.
@@ -205,7 +174,7 @@ jobs:
     # refer:
     #   https://docs.github.com/en/actions/configuring-and-managing-workflows/persisting-workflow-data-using-artifacts#downloading-and-deleting-artifacts-after-a-workflow-run-is-complete
     name: Clean Up Artifacts
-    needs: goreleaser
+    needs: acceptance_test
     # if: ${{ needs.acceptance_test.result == 'success' }}
     runs-on: ubuntu-latest
     steps:

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,9 @@
+[submodule "tests/acceptance/lib/bats-support"]
+	path = tests/acceptance/lib/bats-support
+	url = https://github.com/ztombol/bats-support
+[submodule "tests/acceptance/lib/bats-assert"]
+	path = tests/acceptance/lib/bats-assert
+	url = https://github.com/ztombol/bats-assert
+[submodule "tests/acceptance/lib/bats"]
+	path = tests/acceptance/lib/bats
+	url = https://github.com/sstephenson/bats
diff --git a/internal/collector/execution.go b/internal/collector/execution.go
@@ -55,9 +55,9 @@ func (e *execution) getTiming() types.TimingCollection {
 	// TODO #timing a nice way of doing this
 	res := e.pluginTiming
 	if e.conversionTiming.Operation != "" {
-		res = append(res, e.conversionTiming) //nolint: govet // TODO Timing contains sync.Mutex, find a nice way of handling this
+		res = append(res, e.conversionTiming)
 	}
-	return append(res, e.executionTiming) //nolint: govet // TODO Timing contains sync.Mutex, find a nice way of handling this
+	return append(res, e.executionTiming)
 }
 
 // set state to complete and set end time for the execution and the conversion timing

diff --git a/internal/parquet/duck_db.go b/internal/parquet/duck_db.go
@@ -34,6 +34,7 @@ func newDuckDb() (*duckDb, error) {
 }
 
 func installAndLoadExtensions(db *sql.DB) error {
+	// set the extension directory
 	if _, err := db.Exec(fmt.Sprintf("SET extension_directory = '%s';", filepaths.EnsurePipesDuckDbExtensionsDir())); err != nil {
 		return fmt.Errorf("failed to set extension_directory: %w", err)
 	}

diff --git a/tests/acceptance/lib/bats b/tests/acceptance/lib/bats
diff --git a/tests/acceptance/lib/bats-assert b/tests/acceptance/lib/bats-assert
diff --git a/tests/acceptance/lib/bats-support b/tests/acceptance/lib/bats-support
diff --git a/tests/acceptance/run-local.sh b/tests/acceptance/run-local.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+MY_PATH="`dirname \"$0\"`"              # relative
+MY_PATH="`( cd \"$MY_PATH\" && pwd )`"  # absolutized and normalized
+
+# TODO PSKR review all exports and remove unused ones in tailpipe
+export TAILPIPE_INSTALL_DIR=$(mktemp -d)
+export TZ=UTC
+export WD=$(mktemp -d)
+
+trap "cd -;code=$?;rm -rf $TAILPIPE_INSTALL_DIR; exit $code" EXIT
+
+cd "$WD"
+echo "Working directory: $WD"
+# setup a tailpipe installation
+echo "Install directory: $TAILPIPE_INSTALL_DIR"
+
+# Temporarily disable 'exit on error' since we want to run the collect command and not exit if it fails
+set +e
+tailpipe collect > /dev/null 2>&1
+check_status=$?
+set -e
+
+echo "Installation complete at $TAILPIPE_INSTALL_DIR"
+
+# install chaos plugin
+tailpipe plugin install chaos
+echo "Installed CHAOS plugin"
+
+if [ $# -eq 0 ]; then
+  # Run all test files
+  "$MY_PATH/run.sh"
+else
+  "$MY_PATH/run.sh" "${1}"
+fi
diff --git a/tests/acceptance/run.sh b/tests/acceptance/run.sh
@@ -0,0 +1,59 @@
+#!/bin/bash -e
+
+MY_PATH="`dirname \"$0\"`"              # relative
+MY_PATH="`( cd \"$MY_PATH\" && pwd )`"  # absolutized and normalized
+
+# trap "code=$?;rm -rf $MY_PATH/test_data/templates; exit $code" EXIT
+
+# set this to the source file for development
+export BATS_PATH=$MY_PATH/lib/bats/bin/bats
+export LIB_BATS_ASSERT=$MY_PATH/lib/bats-assert
+export LIB_BATS_SUPPORT=$MY_PATH/lib/bats-support
+export TEST_DATA_DIR=$MY_PATH/test_data/templates
+
+# Must have these commands for the test suite to run
+declare -a required_commands=("sed" "tailpipe" $BATS_PATH "rm" "mv" "cp" "mkdir" "cd" "node" "npm" "npx" "head" "wc" "find" "basename" "dirname")
+
+for required_command in "${required_commands[@]}"
+do
+  if [[ $(command -v $required_command | head -c1 | wc -c) -eq 0 ]]; then
+    echo "$required_command is required for this test suite to run."
+    exit -1
+  fi
+done
+
+# create a copy of the test data templates
+# mkdir $MY_PATH/test_data/templates
+# export TEST_DATA_DIR=$MY_PATH/test_data/templates
+# cp -R $MY_PATH/test_data/templates/* $TEST_DATA_DIR/
+
+# cd $TEST_DATA_DIR
+
+echo " ____  _             _   _               _____         _       "
+echo "/ ___|| |_ __ _ _ __| |_(_)_ __   __ _  |_   _|__  ___| |_ ___ "
+echo "\___ \| __/ _\` | '__| __| | '_ \ / _\` |   | |/ _ \/ __| __/ __|"
+echo " ___) | || (_| | |  | |_| | | | | (_| |   | |  __/\__ \ |_\__ \\"
+echo "|____/ \__\__,_|_|   \__|_|_| |_|\__, |   |_|\___||___/\__|___/"
+echo "                                 |___/                         "
+
+export PATH=$MY_PATH/lib/bats/bin:$PATH
+
+if [[ ! ${TAILPIPE_INSTALL_DIR} ]];
+then
+  export TAILPIPE_INSTALL_DIR="$HOME/.tailpipe"
+fi
+
+batversion=$(bats --version)
+echo $batversion
+echo "Running with TAILPIPE_INSTALL_DIR set to: $TAILPIPE_INSTALL_DIR"
+echo "Running with binary from: $(which tailpipe)"
+
+if [ $# -eq 0 ]; then
+  # Run all test files
+  $BATS_PATH --tap $MY_PATH/test_files
+else
+  $BATS_PATH --tap $MY_PATH/test_files/${1}
+fi
+
+# Setting the exit_code, to use in the github workflow(This only gets set to 0 when the above bats test suite passes)
+echo "::set-output name=exit_code::$(echo $?)"