diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 04c85c0..0e60322 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") \ No newline at end of file +export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") +export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2bbe1dd..b911893 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -69,5 +69,6 @@ steps: - "CI_DATABRICKS_DBT_HOST" - "CI_DATABRICKS_DBT_HTTP_PATH" - "CI_DATABRICKS_DBT_TOKEN" + - "CI_DATABRICKS_DBT_CATALOG" commands: | bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md index 99c450c..0899dc6 100644 --- a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md @@ -12,13 +12,15 @@ Before marking this PR as "ready for review": ## PR Overview **Package version introduced in this PR:** - +- + **This PR addresses the following Issue/Feature(s):** +- **Summary of changes:** - +- ### Submission Checklist - [ ] Alignment meeting with the reviewer (if needed) @@ -27,9 +29,10 @@ Before marking this PR as "ready for review": - [ ] **Validation Steps:** Check for unintentional effects (e.g., add/run consistency & integrity tests) - [ ] **Testing Instructions:** Confirm the change addresses the issue(s) - [ ] **Focus Areas:** Complex logic or queries that need extra attention +- [ ] Merge any relevant open PRs into this PR ### Changelog - [ ] Draft changelog for PR -- [ ] Final changelog for release review \ No newline at end of file +- [ ] Final changelog for release review diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 8ed5853..76d6944 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -3,11 +3,12 @@ on: pull_request: types: - closed - branches: - - main + - labeled jobs: - call-workflow-passing-data: - if: github.event.pull_request.merged - uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main - secrets: inherit \ No newline at end of file + release: + if: | + (github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') || + github.event.label.name == 'pre-release' + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@feature/pre-release-support + secrets: inherit diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml new file mode 100644 index 0000000..1d8c614 --- /dev/null +++ b/.github/workflows/generate-docs.yml @@ -0,0 +1,13 @@ +name: 'generate dbt docs' +on: + pull_request: + types: + - labeled + +jobs: + generate-docs: + if: github.event.label.name == 'docs:ready' + uses: fivetran/dbt_package_automations/.github/workflows/generate-docs.yml@main + secrets: inherit + with: + schema_var_name: github_schema \ No newline at end of file diff --git a/.gitignore b/.gitignore index 184c932..06f4e77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,73 @@ - -target/ +# dbt +**/package-lock.yml +package-lock.yml +.dbt/ dbt_modules/ +dbt_packages/ logs/ -keyfile.json +profiles.yml +target/ +*.log + +# IDE files +.idea/ +.vscode/ +*~ +*.swp +*.swo + +# Jupyter Notebook +.ipynb_checkpoints + +# OS generated files +**/.DS_Store .DS_Store -develop/ -dbt_packages/ +.Spotlight-V100 +.Trashes +._* +Thumbs.db +ehthumbs.db + +# Python +*.egg +*.egg-info/ +*.py[cod] +*.so +*$py.class +.Python +__pycache__/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.env +.installed.cfg +lib/ +lib64/ +MANIFEST +parts/ +sdist/ +var/ +wheels/ + +# Secrets and credentials +.env.* +.secrets +credentials.json +service-account.json + +# Temporary files +.cache/ +*.temp +*.tmp + +# Virtual environments +.conda/ +.env +.venv +ENV/ env/ -package-lock.yml \ No newline at end of file +env.bak/ +venv/ +venv.bak/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 74776bd..3718f32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +[PR #65](https://github.com/fivetran/dbt_github/pull/65) includes the following updates: + +### Under the Hood - July 2025 Updates + +- Updated conditions in `.github/workflows/auto-release.yml`. +- Added `.github/workflows/generate-docs.yml`. +- Added `+docs: show: False` to `integration_tests/dbt_project.yml`. +- Migrated `flags` (e.g., `send_anonymous_usage_stats`, `use_colors`) from `sample.profiles.yml` to `integration_tests/dbt_project.yml`. +- Updated `maintainer_pull_request_template.md` with improved checklist. +- Refreshed README tag block: + - Standardized Quickstart-compatible badge set + - Left-aligned and positioned below the H1 title. +- Updated Python image version to `3.10.13` in `pipeline.yml`. +- Added `CI_DATABRICKS_DBT_CATALOG` to: + - `.buildkite/hooks/pre-command` (as an export) + - `pipeline.yml` (under the `environment` block, after `CI_DATABRICKS_DBT_TOKEN`) +- Added `certifi==2025.1.31` to `requirements.txt` (if missing). +- Updated `.gitignore` to exclude additional DBT, Python, and system artifacts. + # dbt_github v0.9.1 [PR #64](https://github.com/fivetran/dbt_github/pull/64) includes the following updates: diff --git a/README.md b/README.md index 8fffb93..ea47327 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ href="https://github.com/fivetran/dbt_github/blob/main/LICENSE"> - + @@ -67,7 +67,6 @@ packages: Do NOT include the `github_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well. - ### Step 3: Define database and schema variables By default, this package runs using your [destination](https://docs.getdbt.com/docs/running-a-dbt-project/using-the-command-line-interface/configure-your-profile) and the `github` schema. If this is not where your Github data is (for example, if your github schema is named `github_fivetran`), add the following configuration to your root `dbt_project.yml` file: @@ -129,7 +128,7 @@ Fivetran offers the ability for you to orchestrate your dbt project through [Fiv ## Does this package have dependencies? This dbt package is dependent on the following dbt packages. These dependencies are installed by default within this package. For more information on the following packages, refer to the [dbt hub](https://hub.getdbt.com/) site. > IMPORTANT: If you have any of these dependent packages in your own `packages.yml` file, we highly recommend that you remove them from your root `packages.yml` to avoid package version conflicts. - + ```yml packages: - package: fivetran/fivetran_utils @@ -140,7 +139,7 @@ packages: - package: fivetran/github_source version: [">=0.9.0", "<0.10.0"] - + - package: dbt-labs/spark_utils version: [">=0.3.0", "<0.4.0"] ``` diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 8495cce..bc8fb41 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -41,10 +41,10 @@ integration_tests: schema: github_integration_tests_1 threads: 8 databricks: - catalog: null + catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" schema: github_integration_tests_1 - threads: 2 + threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 699f470..2a15ffe 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -67,4 +67,5 @@ dispatch: search_order: ['spark_utils', 'dbt_utils'] flags: - send_anonymous_usage_stats: False \ No newline at end of file + send_anonymous_usage_stats: False + use_colors: True \ No newline at end of file diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 87ae0f9..c9f6024 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -4,4 +4,5 @@ dbt-redshift>=1.3.0,<2.0.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.3.0,<2.0.0 \ No newline at end of file +dbt-databricks>=1.3.0,<2.0.0 +certifi==2025.1.31