Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/bin/bash

set -e

# Export secrets for Docker containers.
# Restrict exposing secrets only to the steps that need them
export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917")
export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917")
export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917")
export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917")
export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917")
export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917")
export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917")
export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917")
export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917")
export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917")
export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917")
export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917")
export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917")
export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917")
export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
export CI_DATABRICKS_SQL_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
export CI_DATABRICKS_SQL_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_TOKEN" --project="dbt-package-testing-363917")
74 changes: 74 additions & 0 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
steps:
- label: ":postgres: Run Tests - Postgres"
key: "run-dbt-postgres"
plugins:
- docker#v3.13.0:
image: "python:3.10.13"
shell: [ "/bin/bash", "-e", "-c" ]
environment:
- "BASH_ENV=/tmp/.bashrc"
- "CI_POSTGRES_DBT_DBNAME"
- "CI_POSTGRES_DBT_HOST"
- "CI_POSTGRES_DBT_PASS"
- "CI_POSTGRES_DBT_USER"
commands: |
bash .buildkite/scripts/run_models.sh postgres

- label: ":snowflake-db: Run Tests - Snowflake"
key: "run_dbt_snowflake"
plugins:
- docker#v3.13.0:
image: "python:3.10.13"
shell: [ "/bin/bash", "-e", "-c" ]
environment:
- "BASH_ENV=/tmp/.bashrc"
- "CI_SNOWFLAKE_DBT_ACCOUNT"
- "CI_SNOWFLAKE_DBT_DATABASE"
- "CI_SNOWFLAKE_DBT_PASS"
- "CI_SNOWFLAKE_DBT_ROLE"
- "CI_SNOWFLAKE_DBT_USER"
- "CI_SNOWFLAKE_DBT_WAREHOUSE"
commands: |
bash .buildkite/scripts/run_models.sh snowflake

- label: ":gcloud: Run Tests - BigQuery"
key: "run_dbt_bigquery"
plugins:
- docker#v3.13.0:
image: "python:3.10.13"
shell: [ "/bin/bash", "-e", "-c" ]
environment:
- "BASH_ENV=/tmp/.bashrc"
- "GCLOUD_SERVICE_KEY"
commands: |
bash .buildkite/scripts/run_models.sh bigquery

- label: ":amazon-redshift: Run Tests - Redshift"
key: "run_dbt_redshift"
plugins:
- docker#v3.13.0:
image: "python:3.10.13"
shell: [ "/bin/bash", "-e", "-c" ]
environment:
- "BASH_ENV=/tmp/.bashrc"
- "CI_REDSHIFT_DBT_DBNAME"
- "CI_REDSHIFT_DBT_HOST"
- "CI_REDSHIFT_DBT_PASS"
- "CI_REDSHIFT_DBT_USER"
commands: |
bash .buildkite/scripts/run_models.sh redshift

- label: ":databricks: Run Tests - Databricks"
key: "run_dbt_databricks"
plugins:
- docker#v3.13.0:
image: "python:3.10.13"
shell: [ "/bin/bash", "-e", "-c" ]
environment:
- "BASH_ENV=/tmp/.bashrc"
- "CI_DATABRICKS_DBT_HOST"
- "CI_DATABRICKS_DBT_HTTP_PATH"
- "CI_DATABRICKS_DBT_TOKEN"
- "CI_DATABRICKS_DBT_CATALOG"
commands: |
bash .buildkite/scripts/run_models.sh databricks
31 changes: 31 additions & 0 deletions .buildkite/scripts/run_models.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/bash

set -euo pipefail

apt-get update
apt-get install libsasl2-dev

python3 -m venv venv
. venv/bin/activate
pip install --upgrade pip setuptools
pip install -r integration_tests/requirements.txt
mkdir -p ~/.dbt
cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml

db=$1
echo `pwd`
cd integration_tests
dbt deps

dbt seed --target "$db" --full-refresh
dbt source freshness --target "$db" || echo "...Only verifying freshness runs…"
dbt compile --target "$db"
dbt run --target "$db" --full-refresh
dbt run --target "$db"
dbt test --target "$db"

# Passthrough metrics test: exercises alias renaming and multi-column passthrough across all delivery models
dbt run --target "$db" --full-refresh --vars '{stackadapt__ad_delivery_passthrough_metrics: [{name: revenue, alias: ad_revenue}, {name: impression_conversions_bigint}], stackadapt__advertiser_delivery_passthrough_metrics: [{name: revenue, alias: advertiser_revenue}], stackadapt__campaign_delivery_passthrough_metrics: [{name: revenue}, {name: impression_conversions_bigint, alias: campaign_view_through_conversions}], stackadapt__campaign_group_delivery_passthrough_metrics: [{name: impression_conversions_bigint, alias: view_through_conversions}]}'
dbt test --target "$db" --vars '{stackadapt__ad_delivery_passthrough_metrics: [{name: revenue, alias: ad_revenue}, {name: impression_conversions_bigint}], stackadapt__advertiser_delivery_passthrough_metrics: [{name: revenue, alias: advertiser_revenue}], stackadapt__campaign_delivery_passthrough_metrics: [{name: revenue}, {name: impression_conversions_bigint, alias: campaign_view_through_conversions}], stackadapt__campaign_group_delivery_passthrough_metrics: [{name: impression_conversions_bigint, alias: view_through_conversions}]}'

dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
113 changes: 113 additions & 0 deletions .github/ISSUE_TEMPLATE/bug-report.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
name: Bug Report
description: Report a bug with this dbt package
labels: ["bug"]
body:
- type: markdown
attributes:
value: |
Thanks for taking the time to fill out this bug report! Please search existing issues before submitting a new one.
- type: textarea
id: description
attributes:
label: Describe the bug
description: A clear and concise description of what the bug is.
validations:
required: true
- type: textarea
id: reproduction
attributes:
label: Steps to reproduce
description: Steps to reproduce the behavior.
placeholder: |
1. Configure package with '...'
2. Run 'dbt run'
3. See error
validations:
required: true
- type: textarea
id: logs
attributes:
label: Relevant error logs
description: Please copy and paste any relevant log output.
render: shell
validations:
required: false
- type: textarea
id: expected
attributes:
label: Expected behavior
description: A clear and concise description of what you expected to happen.
validations:
required: true
- type: textarea
id: solution
attributes:
label: Possible solution
description: If you have a suggestion for how to fix the bug, describe it here.
validations:
required: false
- type: textarea
id: dbt-config
attributes:
label: dbt project config
description: Please provide the relevant sections of your `dbt_project.yml`.
render: yaml
validations:
required: true
- type: textarea
id: packages
attributes:
label: Package versions from packages.yml
description: Please provide your `packages.yml` contents.
render: yaml
validations:
required: true
- type: checkboxes
id: database
attributes:
label: Database type
description: Which database are you using?
options:
- label: BigQuery
- label: Snowflake
- label: Redshift
- label: Databricks
- label: PostgreSQL
validations:
required: true
- type: dropdown
id: orchestration
attributes:
label: Orchestration type
options:
- Quickstart
- Fivetran Transformations for dbt Core
- dbt Cloud
- dbt Core (local)
- Other
validations:
required: true
- type: input
id: dbt-version
attributes:
label: dbt version
placeholder: e.g., 1.8.0
validations:
required: true
- type: textarea
id: additional
attributes:
label: Additional context
description: Add any other context about the problem here.
validations:
required: false
- type: dropdown
id: contribution
attributes:
label: Are you interested in contributing a fix?
options:
- "Yes, I'd like to open a PR"
- "Yes, but I'd need guidance"
- "No"
validations:
required: true
14 changes: 14 additions & 0 deletions .github/ISSUE_TEMPLATE/config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
blank_issues_enabled: false
contact_links:
- name: Share feedback on dbt packages
url: https://www.surveymonkey.com/r/DQ7K7WW
about: We'd love to hear your feedback on our dbt packages.
- name: Fivetran Support Portal
url: https://support.fivetran.com
about: For questions about your Fivetran connector, please contact Fivetran support.
- name: dbt Documentation
url: https://docs.getdbt.com
about: For general dbt questions, consult the dbt documentation.
- name: dbt Community Slack
url: https://www.getdbt.com/community/join-the-community
about: For general dbt questions, join the dbt community Slack.
53 changes: 53 additions & 0 deletions .github/ISSUE_TEMPLATE/feature-request.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
name: Feature Request
description: Suggest an idea or enhancement for this dbt package
labels: ["enhancement"]
body:
- type: markdown
attributes:
value: |
Thanks for taking the time to submit a feature request! Please search existing issues before submitting a new one.
- type: textarea
id: description
attributes:
label: Describe the feature
description: A clear and concise description of the feature or enhancement you'd like to see.
validations:
required: true
- type: textarea
id: rationale
attributes:
label: Why is this feature needed?
description: Explain the use case or problem this feature would solve.
validations:
required: true
- type: textarea
id: implementation
attributes:
label: Proposed implementation
description: If you have a specific implementation in mind, describe it here with any relevant SQL or YAML examples.
validations:
required: true
- type: textarea
id: alternatives
attributes:
label: Alternatives considered
description: Describe any alternative solutions or features you've considered.
validations:
required: false
- type: dropdown
id: contribution
attributes:
label: Are you interested in contributing this feature?
options:
- "Yes, I'd like to open a PR"
- "Yes, but I'd need guidance"
- "No"
validations:
required: true
- type: textarea
id: additional
attributes:
label: Additional context
description: Add any other context, screenshots, or examples here.
validations:
required: false
38 changes: 38 additions & 0 deletions .github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
<!--
Pre-Submission Reminders
Before marking this PR as "ready for review":

- `dbt run --full-refresh && dbt test`
- `dbt run` && `dbt test` (if incremental models are present)
- The related issue is linked, tagged, and appropriately assigned
- Documentation and version updates are included, if applicable
- `docs` have been regenerated (unless there are no code or YAML changes)
- BuildKite integration tests are passing
-->

## PR Overview
**Package version introduced in this PR:**
-

**This PR addresses the following Issue/Feature(s):**
<!-- Add Issue # or internal ticket reference -->
-

**Summary of changes:**
<!-- 1-2 sentences describing PR changes. -->
-

### Submission Checklist
- [ ] Alignment meeting with the reviewer (if needed)
- [ ] Timeline and validation requirements discussed
- [ ] Provide validation details:
- [ ] **Validation Steps:** Check for unintentional effects (e.g., add/run consistency & integrity tests)
- [ ] **Testing Instructions:** Confirm the change addresses the issue(s)
- [ ] **Focus Areas:** Complex logic or queries that need extra attention
- [ ] Merge any relevant open PRs into this PR

### Changelog
<!-- Recommend drafting changelog notes, then refining via ChatGPT using:
"Draft a changelog entry based on the following notes." -->
- [ ] Draft changelog for PR
- [ ] Final changelog for release review
39 changes: 39 additions & 0 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
**Please provide your name and company**

**Link the issue/feature request which this PR is meant to address**
<!--- If an issue was not created, please create one first so we may discuss the PR prior to opening one. -->

**Detail what changes this PR introduces and how this addresses the issue/feature request linked above.**

**How did you validate the changes introduced within this PR?**

**Which warehouse did you use to develop these changes?**

**Did you update the CHANGELOG?**
<!--- Please update the new package version's CHANGELOG entry detailing the changes included in this PR. -->
<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
- [ ] Yes

**Did you update the dbt_project.yml files with the version upgrade (please leverage standard semantic versioning)? (In both your main project and integration_tests)**
<!--- The dbt_project.yml and the integration_tests/dbt_project.yml files contain the version number. Be sure to upgrade it accordingly -->
<!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
- [ ] Yes

**Typically there are additional maintenance changes required before this will be ready for an upcoming release. Are you comfortable with the Fivetran team making a few commits directly to your branch?**
<!--- If you select Yes this will help expedite your PR in case there are small changes required before approval. We encourage you not to use this branch in a production environment as we may make additional updates. -->
<!--- If you select No, we will not make any changes directly to your branch and will either communicate any planned changes via the PR thread or will merge your PR into a separate branch so we may make changes without modifying your branch.. -->
- [ ] Yes
- [ ] No

**If you had to summarize this PR in an emoji, which would it be?**
<!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908) -->
:dancer:

**Feedback**

We are so excited you decided to contribute to the Fivetran community dbt package! We continue to work to improve the packages and would greatly appreciate your [feedback](https://www.surveymonkey.com/r/DQ7K7WW) on our existing dbt packages or what you'd like to see next.

**PR Template**
- [Community Pull Request Template](?expand=1&template=pull_request_template.md) (default)

- [Maintainer Pull Request Template](?expand=1&template=maintainer_pull_request_template.md) (to be used by maintainers)
Loading
Loading