From 4eeaf3127c5df03a4e4973b2a093720a6f673db5 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Wed, 25 Jun 2025 22:39:29 -0500 Subject: [PATCH 1/2] feature/freshness-updates --- .buildkite/pipeline.yml | 12 +++++------ .buildkite/scripts/run_models.sh | 1 + CHANGELOG.md | 26 ++++++++++++++++++++++++ README.md | 1 + dbt_project.yml | 2 +- integration_tests/ci/sample.profiles.yml | 6 +----- integration_tests/dbt_project.yml | 5 ++++- integration_tests/requirements.txt | 16 +++++++-------- models/src_recurly.yml | 7 ++++--- 9 files changed, 52 insertions(+), 24 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 281f1d4..9387b23 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,7 +3,7 @@ steps: key: "run-dbt-postgres" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -18,7 +18,7 @@ steps: key: "run_dbt_snowflake" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -35,7 +35,7 @@ steps: key: "run_dbt_bigquery" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -47,7 +47,7 @@ steps: key: "run_dbt_redshift" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -62,7 +62,7 @@ steps: key: "run_dbt_databricks" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -71,4 +71,4 @@ steps: - "CI_DATABRICKS_DBT_TOKEN" - "CI_DATABRICKS_DBT_CATALOG" commands: | - bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file + bash .buildkite/scripts/run_models.sh databricks diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index ea68415..a204541 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -17,6 +17,7 @@ echo `pwd` cd integration_tests dbt deps dbt seed --target "$db" --full-refresh +dbt source freshness --target "$db" || echo "...Only verifying freshness runs..." dbt run --target "$db" --full-refresh dbt test --target "$db" dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" diff --git a/CHANGELOG.md b/CHANGELOG.md index 5460519..26b3ef0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,29 @@ +# dbt_recurly_source v0.4.0 + +[PR #21](https://github.com/fivetran/dbt_recurly_source/pull/21) includes the following updates: + +## Breaking Change for dbt Core < 1.9.6 + +> *Note: This is not relevant to Fivetran Quickstart users.* + +Migrated `freshness` from a top-level source property to a source `config` in alignment with [recent updates](https://github.com/dbt-labs/dbt-core/issues/11506) from dbt Core. This will resolve the following deprecation warning that users running dbt >= 1.9.6 may have received: + +``` +[WARNING]: Deprecated functionality +Found `freshness` as a top-level property of `recurly` in file +`models/src_recurly.yml`. The `freshness` top-level property should be moved +into the `config` of `recurly`. +``` + +**IMPORTANT:** Users running dbt Core < 1.9.6 will not be able to utilize freshness tests in this release or any subsequent releases, as older versions of dbt will not recognize freshness as a source `config` and therefore not run the tests. + +If you are using dbt Core < 1.9.6 and want to continue running Recurly freshness tests, please elect **one** of the following options: + 1. (Recommended) Upgrade to dbt Core >= 1.9.6 + 2. Do not upgrade your installed version of the `recurly_source` package. Pin your dependency on v0.3.1 in your `packages.yml` file. + 3. Utilize a dbt [override](https://docs.getdbt.com/reference/resource-properties/overrides) to overwrite the package's `recurly` source and apply freshness via the previous release top-level property route. This will require you to copy and paste the entirety of the previous release `src_recurly.yml` file and add an `overrides: recurly_source` property. + +## Under the Hood +- Updates to ensure integration tests use latest version of dbt. # dbt_recurly_source v0.3.1 This release introduces the following updates: diff --git a/README.md b/README.md index c256f58..b84c9dd 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ - Materializes [Recurly staging tables](https://fivetran.github.io/dbt_recurly_source/#!/overview/github_source/models/?g_v=1) which leverages data in the format described by [this ERD](https://fivetran.com/docs/applications/recurly#schemainformation). These staging tables clean, test, and prepare your Recurly data from [Fivetran's connector](https://fivetran.com/docs/applications/recurly) for analysis by doing the following: - Name columns for consistency across all packages and easier analysis. - Adds freshness tests to source data. + - dbt Core >= 1.9.6 is required to run freshness tests out of the box. - Adds column-level testing where applicable. For example, all primary keys are tested for uniqueness and non-null values. - Generates a comprehensive data dictionary of your Recurly data through the [dbt docs site](https://fivetran.github.io/dbt_recurly_source/). - These tables are designed to work simultaneously with our [Recurly transformation package](https://github.com/fivetran/dbt_recurly). diff --git a/dbt_project.yml b/dbt_project.yml index c81d493..48a6bb4 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: "recurly_source" -version: "0.3.1" +version: "0.4.0" config-version: 2 require-dbt-version: [">=1.3.0", "<2.0.0"] vars: diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 93581d1..c51fc0b 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -2,10 +2,6 @@ # HEY! This file is used in the dbt package integrations tests with Buildkite. # You should __NEVER__ check credentials into version control. Thanks for reading :) -config: - send_anonymous_usage_stats: False - use_colors: True - integration_tests: target: redshift outputs: @@ -51,4 +47,4 @@ integration_tests: schema: recurly_source_integration_tests threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" - type: databricks \ No newline at end of file + type: databricks diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 4d8b613..b804692 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'recurly_source_integration_tests' -version: '0.3.1' +version: '0.4.0' profile: 'integration_tests' config-version: 2 @@ -96,3 +96,6 @@ seeds: transaction_data: +column_types: amount: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" + +flags: + send_anonymous_usage_stats: False diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index c071a4d..24f0f78 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,8 +1,8 @@ -dbt-snowflake>=1.3.0,<1.8.0 -dbt-bigquery>=1.3.0,<1.8.0 -dbt-redshift>=1.3.0,<1.8.0 -dbt-postgres>=1.3.0,<1.8.0 -dbt-spark>=1.3.0,<1.8.0 -dbt-spark[PyHive]>=1.3.0,<1.8.0 -dbt-databricks>=1.7.0,<1.8.0 -certifi==2025.1.31 \ No newline at end of file +dbt-snowflake>=1.3.0,<2.0.0 +dbt-bigquery>=1.3.0,<2.0.0 +dbt-redshift>=1.3.0,<2.0.0 +dbt-postgres>=1.3.0,<2.0.0 +dbt-spark>=1.3.0,<2.0.0 +dbt-spark[PyHive]>=1.3.0,<2.0.0 +dbt-databricks>=1.7.0,<2.0.0 +certifi==2025.1.31 diff --git a/models/src_recurly.yml b/models/src_recurly.yml index 0c89321..bec687e 100644 --- a/models/src_recurly.yml +++ b/models/src_recurly.yml @@ -7,9 +7,10 @@ sources: loader: fivetran loaded_at_field: _fivetran_synced - freshness: - warn_after: { count: 72, period: hour } - error_after: { count: 168, period: hour } + config: + freshness: + warn_after: { count: 72, period: hour } + error_after: { count: 168, period: hour } tables: - name: account_balance_history From aa53310ef3e17b2c7a0852eeb98065f587e3e9b7 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Wed, 25 Jun 2025 22:41:32 -0500 Subject: [PATCH 2/2] changelog fix --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26b3ef0..15bc2e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ If you are using dbt Core < 1.9.6 and want to continue running Recurly freshness ## Under the Hood - Updates to ensure integration tests use latest version of dbt. + # dbt_recurly_source v0.3.1 This release introduces the following updates: