Feature that added ad squad & campaign in the ad report

CHANGELOG.md

@@ -1,8 +1,25 @@
-# dbt_snapchat_ads version.version
+# dbt_snapchat_ads v0.8.0
+This release introduces the following updates:
+
+## Breaking Changes
+- Adds ad squad and campaign details to the `snapchat_ads__ad_report` model, so that they can eventually be incorporated in the `dbt_ad_reporting` package for the `ad_reporting__ad_report` model. Fields added are included below. ([#33](https://github.com/fivetran/dbt_snapchat_ads/pull/33))
+  - `ad_squad_id`
+  - `ad_squad_name`
+  - `campaign_id`
+  - `campaign_name`
+
+## Under The Hood
+- Updated the tests to include `ad_squad_id` and `campaign_id` in `snapchat.yml` for `snapchat_ads__ad_report` and `snapchat_ads__url_report` to more closely align with other ad reporting packages. ([#33](https://github.com/fivetran/dbt_snapchat_ads/pull/33))
+- Added integrity test to ensure the `snapchat_ads__ad_report` maintains the same row grain count for `ad_id` and `date_day` betweens the source and end models. ([#33](https://github.com/fivetran/dbt_snapchat_ads/pull/33)) 
+- Updated validation tests in the `integration_tests` folder to check for discrepancies between `conversions` and `conversions_value`. ([#33](https://github.com/fivetran/dbt_snapchat_ads/pull/33)) 
 
 ## Documentation
 - Added Quickstart model counts to README. ([#30](https://github.com/fivetran/dbt_snapchat_ads/pull/30))
 - Corrected references to connectors and connections in the README. ([#30](https://github.com/fivetran/dbt_snapchat_ads/pull/30))
+- Updated `snapchat.yml` with new above field additions. ([#33](https://github.com/fivetran/dbt_snapchat_ads/pull/33))
+
+## Contributors
+- [@JeremyDOwens](https://github.com/JeremyDOwens) ([#32](https://github.com/fivetran/dbt_snapchat_ads/pull/32))
 
 # dbt_snapchat_ads v0.7.0
 [PR #28](https://github.com/fivetran/dbt_snapchat_ads/pull/28) includes the following **BREAKING CHANGE** updates:

README.md

@@ -55,7 +55,7 @@ If you are not using the [Ad Reporting](https://github.com/fivetran/dbt_ad_repor
 ```yml
 packages:
   - package: fivetran/snapchat_ads
-    version: [">=0.7.0", "<0.8.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.8.0", "<0.9.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 Do NOT include the `snapchat_ads_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well.

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'snapchat_ads'
-version: '0.6.2'
+version: '0.8.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:

integration_tests/dbt_project.yml

@@ -1,9 +1,9 @@
 name: 'snapchat_ads_integration_tests'
-version: '0.7.0'
+version: '0.8.0'
 profile: 'integration_tests'
 config-version: 2
 
-vars: 
+vars:
   snapchat_ads_source:
     snapchat_ads_schema: snapchat_ads_integration_tests_6
     snapchat_ads_ad_account_history_identifier:  "snapchat_ad_account_history_data"

integration_tests/tests/consistency/consistency_account_report.sql

@@ -8,9 +8,9 @@ with prod as (
         ad_account_id,
         sum(swipes) as swipes, 
         sum(impressions) as impressions,
-        sum(spend) as spend
-        {# sum(total_conversions) as total_conversions,
-        sum(conversion_purchases_value) as conversion_purchases_value #}
+        sum(spend) as spend,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_purchases_value) as conversion_purchases_value
     from {{ target.schema }}_snapchat_ads_prod.snapchat_ads__account_report
     group by 1
 ),
@@ -20,9 +20,9 @@ dev as (
         ad_account_id,
         sum(swipes) as swipes, 
         sum(impressions) as impressions,
-        sum(spend) as spend
-        {# sum(total_conversions) as total_conversions,
-        sum(conversion_purchases_value) as conversion_purchases_value #}
+        sum(spend) as spend,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_purchases_value) as conversion_purchases_value
     from {{ target.schema }}_snapchat_ads_dev.snapchat_ads__account_report
     group by 1
 ),
@@ -35,11 +35,11 @@ final as (
         prod.impressions as prod_impressions,
         dev.impressions as dev_impressions,
         prod.spend as prod_spend,
-        dev.spend as dev_spend
-        {# prod.total_conversions as prod_total_conversions,
+        dev.spend as dev_spend,
+        prod.total_conversions as prod_total_conversions,
         dev.total_conversions as dev_total_conversions,
         prod.conversion_purchases_value as prod_conversion_purchases_value,
-        dev.conversion_purchases_value as dev_conversion_purchases_value #}
+        dev.conversion_purchases_value as dev_conversion_purchases_value
     from prod
     full outer join dev 
         on dev.ad_account_id = prod.ad_account_id
@@ -51,5 +51,5 @@ where
     abs(prod_swipes - dev_swipes) >= .01
     or abs(prod_impressions - dev_impressions) >= .01
     or abs(prod_spend - dev_spend) >= .01
-    {# or abs(prod_total_conversions - dev_total_conversions) >= .01
-    or abs(prod_conversion_purchases_value - dev_conversion_purchases_value) >= .01 #}
+    or abs(prod_total_conversions - dev_total_conversions) >= .01
+    or abs(prod_conversion_purchases_value - dev_conversion_purchases_value) >= .01

integration_tests/tests/consistency/consistency_ad_squad_report.sql

@@ -8,9 +8,9 @@ with prod as (
         ad_squad_id,
         sum(swipes) as swipes, 
         sum(impressions) as impressions,
-        sum(spend) as spend
-        {# sum(total_conversions) as total_conversions,
-        sum(conversion_purchases_value) as conversion_purchases_value #}
+        sum(spend) as spend,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_purchases_value) as conversion_purchases_value
     from {{ target.schema }}_snapchat_ads_prod.snapchat_ads__ad_squad_report
     group by 1
 ),
@@ -20,9 +20,9 @@ dev as (
         ad_squad_id,
         sum(swipes) as swipes, 
         sum(impressions) as impressions,
-        sum(spend) as spend
-        {# sum(total_conversions) as total_conversions,
-        sum(conversion_purchases_value) as conversion_purchases_value #}
+        sum(spend) as spend,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_purchases_value) as conversion_purchases_value
     from {{ target.schema }}_snapchat_ads_dev.snapchat_ads__ad_squad_report
     group by 1
 ),
@@ -35,11 +35,11 @@ final as (
         prod.impressions as prod_impressions,
         dev.impressions as dev_impressions,
         prod.spend as prod_spend,
-        dev.spend as dev_spend
-        {# prod.total_conversions as prod_total_conversions,
+        dev.spend as dev_spend,
+        prod.total_conversions as prod_total_conversions,
         dev.total_conversions as dev_total_conversions,
         prod.conversion_purchases_value as prod_conversion_purchases_value,
-        dev.conversion_purchases_value as dev_conversion_purchases_value #}
+        dev.conversion_purchases_value as dev_conversion_purchases_value
     from prod
     full outer join dev 
         on dev.ad_squad_id = prod.ad_squad_id
@@ -51,5 +51,5 @@ where
     abs(prod_swipes - dev_swipes) >= .01
     or abs(prod_impressions - dev_impressions) >= .01
     or abs(prod_spend - dev_spend) >= .01
-    {# or abs(prod_total_conversions - dev_total_conversions) >= .01
-    or abs(prod_conversion_purchases_value - dev_conversion_purchases_value) >= .01 #}
+    or abs(prod_total_conversions - dev_total_conversions) >= .01
+    or abs(prod_conversion_purchases_value - dev_conversion_purchases_value) >= .01

integration_tests/tests/consistency/consistency_ads_report.sql

@@ -8,9 +8,9 @@ with prod as (
         ad_id,
         sum(swipes) as swipes, 
         sum(impressions) as impressions,
-        sum(spend) as spend
-        {# sum(total_conversions) as total_conversions,
-        sum(conversion_purchases_value) as conversion_purchases_value #}
+        sum(spend) as spend,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_purchases_value) as conversion_purchases_value
     from {{ target.schema }}_snapchat_ads_prod.snapchat_ads__ad_report
     group by 1
 ),
@@ -20,9 +20,9 @@ dev as (
         ad_id,
         sum(swipes) as swipes, 
         sum(impressions) as impressions,
-        sum(spend) as spend
-        {# sum(total_conversions) as total_conversions,
-        sum(conversion_purchases_value) as conversion_purchases_value #}
+        sum(spend) as spend,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_purchases_value) as conversion_purchases_value
     from {{ target.schema }}_snapchat_ads_dev.snapchat_ads__ad_report
     group by 1
 ),
@@ -35,11 +35,11 @@ final as (
         prod.impressions as prod_impressions,
         dev.impressions as dev_impressions,
         prod.spend as prod_spend,
-        dev.spend as dev_spend
-        {# prod.total_conversions as prod_total_conversions,
+        dev.spend as dev_spend,
+        prod.total_conversions as prod_total_conversions,
         dev.total_conversions as dev_total_conversions,
         prod.conversion_purchases_value as prod_conversion_purchases_value,
-        dev.conversion_purchases_value as dev_conversion_purchases_value #}
+        dev.conversion_purchases_value as dev_conversion_purchases_value
     from prod
     full outer join dev 
         on dev.ad_id = prod.ad_id
@@ -51,5 +51,5 @@ where
     abs(prod_swipes - dev_swipes) >= .01
     or abs(prod_impressions - dev_impressions) >= .01
     or abs(prod_spend - dev_spend) >= .01
-    {# or abs(prod_total_conversions - dev_total_conversions) >= .01
-    or abs(prod_conversion_purchases_value - dev_conversion_purchases_value) >= .01 #}
+    or abs(prod_total_conversions - dev_total_conversions) >= .01
+    or abs(prod_conversion_purchases_value - dev_conversion_purchases_value) >= .01

integration_tests/tests/consistency/consistency_campaign_report.sql

@@ -8,9 +8,9 @@ with prod as (
         campaign_id,
         sum(swipes) as swipes, 
         sum(impressions) as impressions,
-        sum(spend) as spend
-        {# sum(total_conversions) as total_conversions,
-        sum(conversion_purchases_value) as conversion_purchases_value #}
+        sum(spend) as spend,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_purchases_value) as conversion_purchases_value
     from {{ target.schema }}_snapchat_ads_prod.snapchat_ads__campaign_report
     group by 1
 ),
@@ -20,9 +20,9 @@ dev as (
         campaign_id,
         sum(swipes) as swipes, 
         sum(impressions) as impressions,
-        sum(spend) as spend
-        {# sum(total_conversions) as total_conversions,
-        sum(conversion_purchases_value) as conversion_purchases_value #}
+        sum(spend) as spend,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_purchases_value) as conversion_purchases_value
     from {{ target.schema }}_snapchat_ads_dev.snapchat_ads__campaign_report
     group by 1
 ),
@@ -35,11 +35,11 @@ final as (
         prod.impressions as prod_impressions,
         dev.impressions as dev_impressions,
         prod.spend as prod_spend,
-        dev.spend as dev_spend
-        {# prod.total_conversions as prod_total_conversions,
+        dev.spend as dev_spend,
+        prod.total_conversions as prod_total_conversions,
         dev.total_conversions as dev_total_conversions,
         prod.conversion_purchases_value as prod_conversion_purchases_value,
-        dev.conversion_purchases_value as dev_conversion_purchases_value #}
+        dev.conversion_purchases_value as dev_conversion_purchases_value
     from prod
     full outer join dev 
         on dev.campaign_id = prod.campaign_id
@@ -51,5 +51,5 @@ where
     abs(prod_swipes - dev_swipes) >= .01
     or abs(prod_impressions - dev_impressions) >= .01
     or abs(prod_spend - dev_spend) >= .01
-    {# or abs(prod_total_conversions - dev_total_conversions) >= .01
-    or abs(prod_conversion_purchases_value - dev_conversion_purchases_value) >= .01 #}
+    or abs(prod_total_conversions - dev_total_conversions) >= .01
+    or abs(prod_conversion_purchases_value - dev_conversion_purchases_value) >= .01

integration_tests/tests/consistency/consistency_url_report.sql

@@ -8,9 +8,9 @@ with prod as (
         ad_id,
         sum(swipes) as swipes, 
         sum(impressions) as impressions,
-        sum(spend) as spend
-        {# sum(total_conversions) as total_conversions,
-        sum(conversion_purchases_value) as conversion_purchases_value #}
+        sum(spend) as spend,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_purchases_value) as conversion_purchases_value
     from {{ target.schema }}_snapchat_ads_prod.snapchat_ads__url_report
     group by 1
 ),
@@ -20,9 +20,9 @@ dev as (
         ad_id,
         sum(swipes) as swipes, 
         sum(impressions) as impressions,
-        sum(spend) as spend
-        {# sum(total_conversions) as total_conversions,
-        sum(conversion_purchases_value) as conversion_purchases_value #}
+        sum(spend) as spend,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_purchases_value) as conversion_purchases_value
     from {{ target.schema }}_snapchat_ads_dev.snapchat_ads__url_report
     group by 1
 ),
@@ -35,11 +35,11 @@ final as (
         prod.impressions as prod_impressions,
         dev.impressions as dev_impressions,
         prod.spend as prod_spend,
-        dev.spend as dev_spend
-        {# prod.total_conversions as prod_total_conversions,
+        dev.spend as dev_spend,
+        prod.total_conversions as prod_total_conversions,
         dev.total_conversions as dev_total_conversions,
         prod.conversion_purchases_value as prod_conversion_purchases_value,
-        dev.conversion_purchases_value as dev_conversion_purchases_value #}
+        dev.conversion_purchases_value as dev_conversion_purchases_value
     from prod
     full outer join dev 
         on dev.ad_id = prod.ad_id
@@ -51,5 +51,5 @@ where
     abs(prod_swipes - dev_swipes) >= .01
     or abs(prod_impressions - dev_impressions) >= .01
     or abs(prod_spend - dev_spend) >= .01
-    {# or abs(prod_total_conversions - dev_total_conversions) >= .01
-    or abs(prod_conversion_purchases_value - dev_conversion_purchases_value) >= .01 #}
+    or abs(prod_total_conversions - dev_total_conversions) >= .01
+    or abs(prod_conversion_purchases_value - dev_conversion_purchases_value) >= .01

integration_tests/tests/integrity/vertical_ad_grain_row_comparison.sql

@@ -0,0 +1,40 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with ad_day_source as (
+
+    select 
+        ad_id, 
+        cast(date_hour as date) as date_day_source
+    from {{ ref('stg_snapchat_ads__ad_hourly_report') }}
+    {{ dbt_utils.group_by(2) }}
+), 
+
+ad_day_end as (
+
+    select 
+        ad_id, 
+        date_day as date_day_end
+    from {{ ref('snapchat_ads__ad_report') }}
+
+    {{ dbt_utils.group_by(2) }}
+),
+
+final as (
+    -- test will fail if any rows from source not found in end
+    (select * from ad_day_source
+    except distinct
+    select * from ad_day_end)
+
+    union all -- union since we only care if rows are produced
+
+    -- test will fail if any rows from end are not found in source
+    (select * from ad_day_end
+    except distinct
+    select * from ad_day_source)
+    )
+
+select *
+from final

models/snapchat.yml

@@ -46,6 +46,8 @@ models:
           combination_of_columns:
             - source_relation
             - ad_id
+            - ad_squad_id
+            - campaign_id
             - date_day
     columns:
       - name: source_relation
@@ -64,6 +66,14 @@ models:
         description: The name of the account in Snapchat.
       - name: ad_name
         description: The name of the ad in Snapchat.
+      - name: ad_squad_id
+        description: The ID of the ad squad in Snapchat.
+      - name: ad_squad_name
+        description: The name of the ad squad in Snapchat.
+      - name: campaign_id
+        description: The ID of the campaign in Snapchat.
+      - name: campaign_name
+        description: The name of the campaign in Snapchat.
       - name: currency
         description: The current used by the account in Snapchat.
       - name: spend
@@ -171,6 +181,8 @@ models:
           combination_of_columns:
             - source_relation
             - ad_id
+            - ad_squad_id
+            - campaign_id
             - date_day
           config:
             # Below is so we don't run this test if the user has allowed urls to have null entries