diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command deleted file mode 100644 index 81b7b00..0000000 --- a/.buildkite/hooks/pre-command +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -e - -# Export secrets for Docker containers. -# Restrict exposing secrets only to the steps that need them -export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 9387b23..0000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,74 +0,0 @@ -steps: - - label: ":postgres: Run Tests - Postgres" - key: "run-dbt-postgres" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_POSTGRES_DBT_DBNAME" - - "CI_POSTGRES_DBT_HOST" - - "CI_POSTGRES_DBT_PASS" - - "CI_POSTGRES_DBT_USER" - commands: | - bash .buildkite/scripts/run_models.sh postgres - - - label: ":snowflake-db: Run Tests - Snowflake" - key: "run_dbt_snowflake" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_SNOWFLAKE_DBT_ACCOUNT" - - "CI_SNOWFLAKE_DBT_DATABASE" - - "CI_SNOWFLAKE_DBT_PASS" - - "CI_SNOWFLAKE_DBT_ROLE" - - "CI_SNOWFLAKE_DBT_USER" - - "CI_SNOWFLAKE_DBT_WAREHOUSE" - commands: | - bash .buildkite/scripts/run_models.sh snowflake - - - label: ":gcloud: Run Tests - BigQuery" - key: "run_dbt_bigquery" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "GCLOUD_SERVICE_KEY" - commands: | - bash .buildkite/scripts/run_models.sh bigquery - - - label: ":amazon-redshift: Run Tests - Redshift" - key: "run_dbt_redshift" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_REDSHIFT_DBT_DBNAME" - - "CI_REDSHIFT_DBT_HOST" - - "CI_REDSHIFT_DBT_PASS" - - "CI_REDSHIFT_DBT_USER" - commands: | - bash .buildkite/scripts/run_models.sh redshift - - - label: ":databricks: Run Tests - Databricks" - key: "run_dbt_databricks" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_DATABRICKS_DBT_HOST" - - "CI_DATABRICKS_DBT_HTTP_PATH" - - "CI_DATABRICKS_DBT_TOKEN" - - "CI_DATABRICKS_DBT_CATALOG" - commands: | - bash .buildkite/scripts/run_models.sh databricks diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh deleted file mode 100644 index 42dda6b..0000000 --- a/.buildkite/scripts/run_models.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -apt-get update -apt-get install libsasl2-dev - -python3 -m venv venv -. venv/bin/activate -pip install --upgrade pip setuptools -pip install -r integration_tests/requirements.txt -mkdir -p ~/.dbt -cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml - -db=$1 -echo `pwd` -cd integration_tests -dbt deps -dbt seed --target "$db" --full-refresh -dbt run --target "$db" --full-refresh -dbt test --target "$db" -dbt run --vars '{zuora__standardized_billing_model_enabled: false, zuora__using_credit_balance_adjustment: false, zuora__using_taxation_item: false, zuora__using_refund: false, zuora__using_refund_invoice_payment: false}' --target "$db" --full-refresh -dbt test --target "$db" - -dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" - diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml deleted file mode 100644 index e085000..0000000 --- a/integration_tests/ci/sample.profiles.yml +++ /dev/null @@ -1,50 +0,0 @@ - -# HEY! This file is used in the dbt package integrations tests with Buildkite. -# You should __NEVER__ check credentials into version control. Thanks for reading :) - -integration_tests: - target: redshift - outputs: - redshift: - type: redshift - host: "{{ env_var('CI_REDSHIFT_DBT_HOST') }}" - user: "{{ env_var('CI_REDSHIFT_DBT_USER') }}" - pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" - dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" - port: 5439 - schema: zuora_integration_tests_4 - threads: 8 - bigquery: - type: bigquery - method: service-account-json - project: 'dbt-package-testing' - schema: zuora_integration_tests_4 - threads: 8 - keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" - snowflake: - type: snowflake - account: "{{ env_var('CI_SNOWFLAKE_DBT_ACCOUNT') }}" - user: "{{ env_var('CI_SNOWFLAKE_DBT_USER') }}" - password: "{{ env_var('CI_SNOWFLAKE_DBT_PASS') }}" - role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" - database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" - warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: zuora_integration_tests_4 - threads: 8 - postgres: - type: postgres - host: "{{ env_var('CI_POSTGRES_DBT_HOST') }}" - user: "{{ env_var('CI_POSTGRES_DBT_USER') }}" - pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" - dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" - port: 5432 - schema: zuora_integration_tests_4 - threads: 8 - databricks: - catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" - host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" - http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: zuora_integration_tests_4 - threads: 8 - token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" - type: databricks \ No newline at end of file diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml new file mode 100644 index 0000000..d6320b0 --- /dev/null +++ b/integration_tests/ci/test_scenarios.yml @@ -0,0 +1,26 @@ +# Only set if not using default adapter ranges +# dbt_adapter_versions: +# snowflake: ">=1.3.0,<2.0.0" +# bigquery: ">=1.3.0,<2.0.0" +# postgres: ">=1.3.0,<2.0.0" +# redshift: ">=1.3.0,<2.0.0" +# databricks: ">=1.6.0,<2.0.0" + +schema_variable_name: "zuora_schema" + +include_databricks_sql: false +include_sqlserver: false + +test_scenarios: + - name: "Default" + vars: {} + include_incremental: false + + - name: "features disabled" + vars: + zuora__standardized_billing_model_enabled: false + zuora__using_credit_balance_adjustment: false + zuora__using_taxation_item: false + zuora__using_refund: false + zuora__using_refund_invoice_payment: false + include_incremental: false \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 09eac48..578d17f 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -10,7 +10,7 @@ models: +schema: "zuora_{{ var('directed_schema','dev') }}" vars: - zuora_schema: zuora_integration_tests_4 + zuora_schema: zuora_integration_tests zuora: zuora_account_identifier: "zuora_account_data" zuora_amendment_identifier: "zuora_amendment_data" @@ -41,12 +41,11 @@ seeds: +docs: show: False zuora_integration_tests: - +column_types: - _fivetran_synced: "timestamp" - created_date: "timestamp" - updated_date: "timestamp" zuora_account_data: +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" credit_balance: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" last_invoice_date: "timestamp" tax_exempt_effective_date: "timestamp" @@ -58,8 +57,32 @@ seeds: balance: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" bill_cycle_day: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" mrr: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" + zuora_amendment_data: + +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" + current_term: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + renewal_term: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + zuora_contact_data: + +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" + zuora_credit_balance_adjustment_data: + +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" + amount: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" + amount_home_currency: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" + cancelled_on: "timestamp" + exchange_rate: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" zuora_invoice_data: +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" adjustment_amount: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" amount_without_tax: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" amount_without_tax_home_currency: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" @@ -74,7 +97,10 @@ seeds: exchange_rate: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" tax_amount: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" zuora_invoice_item_data: - +column_types: + +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" charge_amount: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" charge_amount_home_currency: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" charge_date: "timestamp" @@ -85,12 +111,23 @@ seeds: tax_amount: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" zuora_invoice_payment_data: +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" amount: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" amount_currency_rounding: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" amount_home_currency: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" exchange_rate: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" + zuora_order_data: + +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" zuora_payment_data: - +column_types: + +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" applied_amount: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" applied_credit_balance_amount: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" settled_on: "timestamp" @@ -102,14 +139,30 @@ seeds: submitted_on: "timestamp" zuora_payment_method_data: +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" last_failed_sale_transaction_date: "timestamp" last_transaction_date_time: "timestamp" max_consecutive_payment_failures: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" num_consecutive_failures: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" total_number_of_error_payments: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" total_number_of_processed_payments: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + zuora_product_data: + +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" + zuora_product_rate_plan_data: + +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" zuora_product_rate_plan_charge_data: +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" default_quantity: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" included_units: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" max_quantity: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" @@ -117,8 +170,16 @@ seeds: specific_billing_period: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" up_to_periods: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" bill_cycle_day: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + zuora_rate_plan_data: + +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" zuora_rate_plan_charge_data: +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" booking_exchange_rate: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" dmrc: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" dmrchome_currency: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" @@ -133,17 +194,27 @@ seeds: mrr: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" quantity: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" zuora_refund_data: - +column_types: + +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" amount: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" refund_transaction_time: "timestamp" refund_date: "timestamp" cancelled_on: "timestamp" submitted_on: "timestamp" + zuora_refund_invoice_payment_data: + +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" zuora_subscription_data: - +column_types: + +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" contract_acceptance_date: "timestamp" contract_effective_date: "timestamp" - created_date: "timestamp" initial_term: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" original_created_date: "timestamp" service_activation_date: "timestamp" @@ -156,21 +227,14 @@ seeds: current_term: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" renewal_term: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" zuora_taxation_item_data: - +column_types: + +column_types: + _fivetran_synced: "timestamp" + created_date: "timestamp" + updated_date: "timestamp" tax_date: "timestamp" balance: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" exchange_rate: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" tax_amount: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" - zuora_credit_balance_adjustment_data: - +column_types: - amount: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" - amount_home_currency: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" - cancelled_on: "timestamp" - exchange_rate: "{{ 'FLOAT64' if target.type == 'bigquery' else 'float' }}" - zuora_amendment_data: - +column_types: - current_term: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - renewal_term: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" flags: send_anonymous_usage_stats: False use_colors: True \ No newline at end of file