[Prerelease] Multicurrency variable config update

.buildkite/scripts/run_models.sh

@@ -19,8 +19,8 @@ dbt deps
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{zuora__using_credit_balance_adjustment: false, zuora__using_taxation_item: false, zuora__using_refund: false, zuora__using_refund_invoice_payment: false}' --target "$db" 
-dbt test --target '{zuora__using_credit_balance_adjustment: false, zuora__using_taxation_item: false, zuora__using_refund: false, zuora__using_refund_invoice_payment: false}' --target "$db"
+dbt run --vars '{zuora__using_credit_balance_adjustment: false, zuora__using_taxation_item: false, zuora__using_refund: false, zuora__using_refund_invoice_payment: false}' --target "$db" --full-refresh
+dbt test --target "$db" 
 
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
 

CHANGELOG.md

@@ -1,4 +1,36 @@
-# dbt_zuora version.version
+# dbt_zuora v0.3.2-a2
+This pre-release introduces the following updates.
+
+## Bug Fixes  
+- Flipped the variable logic in `zuora__using_multicurrency` in these models so that the `*_amount` values are called by default rather than `*_amount_home_currency` when the variable isn't set: ([#21](https://github.com/fivetran/dbt_zuora/pull/21))
+  - `zuora__line_item_history`: For setting the `gross_revenue` and `discount_revenue` values (these values are then used to calculate `net_revenue`).
+  - `zuora__monthly_recurring_revenue`: For setting the `mrr_expected_current_month` value.
+  - `int_zuora__transaction_grouped`: For setting daily amount values for invoices, discounts, taxes and credit balance adjustments to flow downsteram into the `zuora__account_daily_overview` model.
+- Multicurrency customers must set the `zuora__using_multicurrency` variable to `true` to enable that functionality. ([#21](https://github.com/fivetran/dbt_zuora/pull/21))
+
+## Under the Hood
+- Created consistency test within `integration_tests` for the `zuora__monthly_recurring_revenue` model. ([#21](https://github.com/fivetran/dbt_zuora/pull/21))
+- Updated `run_models.sh` to remove duplicative Buildkite script.
+
+# dbt_zuora v0.3.2-a1
+This pre-release introduces the following updates.
+
+## Bug Fixes (originating within the upstream `zuora_source` package):
+- Leveraged the `{{ dbt.type_timestamp() }}` macro within staging models for all timestamp fields. Certain Redshift warehouses sync these fields as `timestamp with time zone` fields by default, causing errors in the `zuora` package. This macro appropriately removes timezone values from the UTC timestamps and ensures successful compilations of these models. 
+- For more details, please refer to the relevant [dbt_zuora_source v0.2.2-a1 release](https://github.com/fivetran/dbt_zuora_source/releases/tag/v0.2.2-a1).
+- Updated `subscription_period_started_at` and `subscription_period_ended_at` with the `{{ dbt.type_timestamp() }}` cast in `zuora__line_item_enhanced` to remove date/timestamp mismatches on the `union all` function. ([#20](https://github.com/fivetran/dbt_zuora/pull/20))
+
+## Under the Hood
+- Replaced the deprecated `dbt.current_timestamp_backcompat()` function with `dbt.current_timestamp()` to ensure all timestamps are captured in UTC.  ([#20](https://github.com/fivetran/dbt_zuora/pull/20))
+- This change is applied in the following end models:
+  - `zuora__billing_history`
+  - `zuora__line_item_history`
+  - `zuora__subscription_overview`
+- As well as the intermediate models:
+  - `int_zuora__account_enriched`
+  - `int_zuora__mrr_date_spine`
+  - `int_zuora__transaction_date_spine`
+- Added consistency tests within `integration_tests` for the `zuora__billing_history` and `zuora__subscription_overview` models. ([#20](https://github.com/fivetran/dbt_zuora/pull/20))
 
 ## Documentation
 - Added Quickstart model counts to README. ([#19](https://github.com/fivetran/dbt_zuora/pull/19))

README.md

@@ -75,7 +75,7 @@ Include the following zuora package version in your `packages.yml` file.
 ```yaml
 packages:
   - package: fivetran/zuora
-    version: [">=0.3.0", "<0.4.0"]
+    version: 0.3.2-a2
 ```
 Do NOT include the `zuora_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well.
 
@@ -202,7 +202,7 @@ This dbt package is dependent on the following dbt packages. These dependencies
 ```yml
 packages:
     - package: fivetran/zuora_source
-      version: [">=0.2.0", "<0.3.0"]
+      version: 0.2.2-a1
 
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'zuora'
-version: '0.3.1'
+version: '0.3.2'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: zuora_integration_tests_1
+      schema: zuora_integration_tests_3
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: zuora_integration_tests_1
+      schema: zuora_integration_tests_3
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: zuora_integration_tests_1
+      schema: zuora_integration_tests_3
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: zuora_integration_tests_1
+      schema: zuora_integration_tests_3
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: zuora_integration_tests_1
+      schema: zuora_integration_tests_3
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,14 +1,16 @@
 name: 'zuora_integration_tests'
-version: '0.3.1'
+version: '0.3.2'
 profile: 'integration_tests'
 config-version: 2
 
-# For use with validations
 models:
   +schema: "zuora_{{ var('directed_schema','dev') }}"
 
 vars:
-  zuora_schema: zuora_integration_tests_1
+  # Uncomment when regenerating docs
+  # zuora__standardized_billing_model_enabled: true
+
+  zuora_schema: zuora_integration_tests_3
   zuora_source:
     zuora_account_identifier: "zuora_account_data"
     zuora_amendment_identifier: "zuora_amendment_data"
@@ -29,7 +31,8 @@ vars:
     zuora_refund_invoice_payment_identifier: "zuora_refund_invoice_payment_data"
     zuora_subscription_identifier: "zuora_subscription_data"
     zuora_taxation_item_identifier: "zuora_taxation_item_data"
-
+
+
 dispatch:
   - macro_namespace: dbt_utils
     search_order: ['spark_utils', 'dbt_utils']

integration_tests/tests/consistency/consistency_billing_history.sql

@@ -0,0 +1,31 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_zuora_prod.zuora__billing_history
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_zuora_dev.zuora__billing_history
+), 
+
+final as (
+    -- test will fail if any rows from prod are not found in dev
+    (select * from prod
+    except distinct
+    select * from dev)
+
+    union all -- union since we only care if rows are produced
+
+    -- test will fail if any rows from dev are not found in prod
+    (select * from dev
+    except distinct
+    select * from prod)
+    )
+
+select *
+from final

integration_tests/tests/consistency/consistency_monthly_recurring_revenue.sql

@@ -0,0 +1,31 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_zuora_prod.zuora__monthly_recurring_revenue
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_zuora_dev.zuora__monthly_recurring_revenue
+), 
+
+final as (
+    -- test will fail if any rows from prod are not found in dev
+    (select * from prod
+    except distinct
+    select * from dev)
+
+    union all -- union since we only care if rows are produced
+
+    -- test will fail if any rows from dev are not found in prod
+    (select * from dev
+    except distinct
+    select * from prod)
+    )
+
+select *
+from final

integration_tests/tests/consistency/consistency_subscription_overview.sql

@@ -0,0 +1,31 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_zuora_prod.zuora__subscription_overview
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_zuora_dev.zuora__subscription_overview
+), 
+
+final as (
+    -- test will fail if any rows from prod are not found in dev
+    (select * from prod
+    except distinct
+    select * from dev)
+
+    union all -- union since we only care if rows are produced
+
+    -- test will fail if any rows from dev are not found in prod
+    (select * from dev
+    except distinct
+    select * from prod)
+    )
+
+select *
+from final

models/intermediate/int_zuora__account_enriched.sql

@@ -1,6 +1,6 @@
 with account as (
 
-    select * 
+    select *
     from {{ var('account') }} 
     where is_most_recent_record 
 ),
@@ -47,8 +47,8 @@ account_details as (
         mrr,
         status,
         auto_pay, 
-        {{ dbt_utils.safe_divide( dbt.datediff("account.created_date", dbt.current_timestamp_backcompat(), "day"), 30) }} as account_active_months,
-        case when {{ dbt.datediff("account.created_date", dbt.current_timestamp_backcompat(), "day") }} <= 30
+        {{ dbt_utils.safe_divide( dbt.datediff("account.created_date", dbt.current_timestamp(), "day"), 30) }} as account_active_months,
+        case when {{ dbt.datediff("account.created_date", dbt.current_timestamp(), "day") }} <= 30
             then true else false end as is_new_customer
 
         {{ fivetran_utils.persist_pass_through_columns('zuora_account_pass_through_columns') }}
@@ -67,7 +67,7 @@ account_totals as (
         max(most_recent_credit_balance_adjustment_date) as most_recent_credit_balance_adjustment_date,
         {% endif %}
 
-        sum(case when cast({{ dbt.date_trunc('day', dbt.current_timestamp_backcompat()) }} as date) > due_date
+        sum(case when cast({{ dbt.date_trunc('day', dbt.current_timestamp()) }} as date) > due_date
                 and invoice_amount != invoice_amount_paid 
                 then invoice_amount_unpaid else 0 end) as total_amount_past_due
 

models/intermediate/int_zuora__mrr_date_spine.sql

@@ -16,7 +16,7 @@ with spine as (
     {# If only compiling, creates range going back 1 year #}
     {% else %} 
         {% set calc_first_date = dbt.dateadd("year", "-1", "current_date") %}
-        {% set calc_last_date = dbt.current_timestamp_backcompat() %}
+        {% set calc_last_date = dbt.current_timestamp() %}
     {% endif %}
 
     {# Prioritizes variables over calculated dates #}

models/intermediate/int_zuora__transaction_date_spine.sql

@@ -17,7 +17,7 @@ with spine as (
     {# If only compiling, creates range going back 1 year #}
     {% else %} 
         {% set calc_first_date = dbt.dateadd("year", "-1", "current_date") %}
-        {% set calc_last_date = dbt.current_timestamp_backcompat() %}
+        {% set calc_last_date = dbt.current_timestamp() %}
     {% endif %}
 
     {# Prioritizes variables over calculated dates #}

models/intermediate/int_zuora__transactions_grouped.sql

@@ -22,9 +22,9 @@ transactions_grouped as (
 
         {% for col in sum_cols %}
         {% if var('zuora__using_multicurrency', false) %}
-            sum({{ col }}) as daily_{{ col }},
-        {% else %} 
             sum({{ col }}_home_currency) as daily_{{ col }},
+        {% else %} 
+            sum({{ col }}) as daily_{{ col }},
         {% endif %}
         {% endfor %}
 

models/standardized_models/zuora__line_item_enhanced.sql

@@ -93,8 +93,8 @@ select
     invoices.refund_amount,
     line_items.subscription_id,
     rate_plan.name as subscription_plan,
-    subscriptions.subscription_start_date as subscription_period_started_at,
-    subscriptions.subscription_end_date as subscription_period_ended_at,
+    cast(subscriptions.subscription_start_date as {{ dbt.type_timestamp() }}) as subscription_period_started_at,
+    cast(subscriptions.subscription_end_date as {{ dbt.type_timestamp() }}) as subscription_period_ended_at,
     subscriptions.status as subscription_status,
     line_items.account_id as customer_id,
     'customer' as customer_level,

models/zuora__billing_history.sql

@@ -23,7 +23,7 @@ with invoice as (
         due_date,
         status,
         source_type as purchase_type,
-        sum(case when cast({{ dbt.date_trunc('day', dbt.current_timestamp_backcompat()) }} as date) > due_date
+        sum(case when cast({{ dbt.date_trunc('day', dbt.current_timestamp()) }} as date) > due_date
                 and amount != payment_amount
                 then balance else 0 end) as total_amount_past_due
     from {{ var('invoice') }} 

models/zuora__line_item_history.sql

@@ -92,13 +92,13 @@ invoice_revenue_items as (
     select
         invoice_item_id, 
         {% if var('zuora__using_multicurrency', false) %}
-        charge_amount as gross_revenue,
-        case when cast(processing_type as {{ dbt.type_string() }})= '1' 
-            then charge_amount else 0 end as discount_revenue
-        {% else %} 
         charge_amount_home_currency as gross_revenue,
         case when cast(processing_type as {{ dbt.type_string() }})= '1' 
             then charge_amount_home_currency else 0 end as discount_revenue
+        {% else %} 
+        charge_amount as gross_revenue,
+        case when cast(processing_type as {{ dbt.type_string() }})= '1' 
+            then charge_amount else 0 end as discount_revenue
         {% endif %}
     from invoice_item_enhanced
 ),

models/zuora__monthly_recurring_revenue.sql

@@ -20,9 +20,9 @@ mrr_by_account as (
         coalesce(month_spine.account_month, line_items.service_start_month) as account_month,
 
         {% if var('zuora__using_multicurrency', false) %}
-        sum(case when charge_mrr is null then 0 else charge_mrr end) as mrr_expected_current_month,
-        {% else %} 
         sum(case when charge_mrr_home_currency is null then 0 else charge_mrr_home_currency end) as mrr_expected_current_month,
+        {% else %} 
+        sum(case when charge_mrr is null then 0 else charge_mrr end) as mrr_expected_current_month,
         {% endif %}
 
         {% set sum_cols = ['gross', 'discount', 'net'] %}

models/zuora__subscription_overview.sql

@@ -89,7 +89,7 @@ subscription_overview as (
             then {{ dbt.datediff('subscription.' ~ date_col ~ '_start_date', 'subscription.' ~ date_col ~ '_end_date', 'day') }} 
             when subscription.term_type = 'EVERGREEN' and subscription.cancelled_date is not null
             then {{ dbt.datediff('subscription.' ~ date_col ~ '_start_date', 'subscription.cancelled_date', 'day') }}
-            else {{ dbt.datediff('subscription.' ~ date_col ~ '_start_date', dbt.current_timestamp_backcompat(), 'day') }}
+            else {{ dbt.datediff('subscription.' ~ date_col ~ '_start_date', dbt.current_timestamp(), 'day') }}
             end as {{ date_col }}_days
         {{ ',' if not loop.last -}}
         {% endfor %}

packages.yml

@@ -1,3 +1,3 @@
 packages:
   - package: fivetran/zuora_source
-    version: [">=0.2.0", "<0.3.0"]
+    version: 0.2.2-a1