Bug/null customer tags (#89)

* bug/null-customer-tags

* update seeds and regen docs

* revisions and regen docs

* review updates

.quickstart/quickstart.yml

@@ -24,6 +24,5 @@ public_models: [
   "shopify__products",
   "shopify__transactions",
   "shopify__customers",
-  "shopify__order_lines",
-  "shopify__line_item_enhanced"
+  "shopify__order_lines"
 ]

CHANGELOG.md

@@ -1,3 +1,13 @@
+# dbt_shopify v0.13.2
+[PR #89](https://github.com/fivetran/dbt_shopify/pull/89) includes the following changes:
+
+## Bug Fixes
+- Fixed an issue where the `shopify__customers` model incorrectly displayed NULL values for the `customer_tags` field for customers without orders. Updated the logic to ensure customer tags are retrieved even if no orders have been placed for that customer.
+
+## Under the Hood
+- Updated seed data to include customers without orders, verifying that their tags are correctly pulled through.
+- Added consistency and integrity tests for the `shopify__customers` model to ensure accurate handling of customer tags for all customers.
+
 # dbt_shopify v0.13.1
 [PR #87](https://github.com/fivetran/dbt_shopify/pull/87) includes the following changes:
 

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'shopify'
-version: '0.13.1'
+version: '0.13.2'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'shopify_integration_tests'
-version: '0.13.1'
+version: '0.13.2'
 profile: 'integration_tests'
 config-version: 2
 
@@ -9,8 +9,11 @@ clean-targets:         # directories to be removed by `dbt clean`
   - "dbt_modules"
 
 vars:
-  shopify_using_fulfillment_event: true # set to true when regenerating docs
-  shopify_using_all_metafields: true # set to true when regenerating docs
+  ## Uncomment when regenerating docs
+  # shopify_using_fulfillment_event: true
+  # shopify_using_all_metafields: true
+  # shopify__standardized_billing_model_enabled: true
+
   shopify_schema: shopify_integration_tests_11
   shopify_source:
     shopify_customer_identifier: "shopify_customer_data"
@@ -67,6 +70,10 @@ seeds:
         id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         default_address_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         email: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}"
+    shopify_customer_tag_data:
+      +column_types:
+        _fivetran_synced: timestamp
+        customer_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
     shopify_order_data:
       +column_types:
         created_at: timestamp

integration_tests/seeds/shopify_customer_data.csv

@@ -1,4 +1,5 @@
 id,first_name,last_name,email,phone,state,orders_count,total_spent,created_at,updated_at,accepts_marketing,tax_exempt,verified_email,default_address_id,_fivetran_synced
 3588998496353,29e00d3659d1c5e75f99e892f0c1a1f1,3f0e6a46fb84eb1e6f5f00d86aa53b1b,ab0bf25ab8b2a6b78af26a141dd6f455,,disabled,0,0,2020-09-11 13:26:15.000 ,2020-09-11 13:26:15.000 ,FALSE,FALSE,TRUE,3951726461025,2020-09-12 00:14:04.512 
 3589760876641,f0962b7a185488ecb752cedac1038349,aa35cb67c26e64bb81a1bf3f17e858ba,021cb20b5c78751fc7ddc091b6b69b3e,,invited,1,2.8,2020-09-11 19:35:42.000 ,2020-09-11 19:41:04.000 ,TRUE,FALSE,TRUE,3952669655137,2020-09-12 00:14:04.506 
-3584045351009,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,dce90c7b4e52e045e5975836aff49cf1,,disabled,2,9.18,2020-09-09 22:57:44.000 ,2020-09-09 23:01:55.000 ,FALSE,FALSE,TRUE,3946055729249,2020-09-10 00:13:59.106 
+3584045351009,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,dce90c7b4e52e045e5975836aff49cf1,,disabled,2,9.18,2020-09-09 22:57:44.000 ,2020-09-09 23:01:55.000 ,FALSE,FALSE,TRUE,3946055729249,2020-09-10 00:13:59.106 
+3584045351010,d3bae70c9d49bb7cb5a74cdd0eae7fc4,0dd89cff60965dff8f9ea2bc952a5474,dce90c7b4e52e045e5975836aff49cf1,,disabled,2,9.18,2020-09-09 22:57:44.000 ,2020-09-09 23:01:55.000 ,FALSE,FALSE,TRUE,3946055729249,2020-09-10 00:13:59.106 

integration_tests/seeds/shopify_customer_tag_data.csv

@@ -1,4 +1,6 @@
 customer_id,index,_fivetran_synced,value
 9919268,1,2022-12-03 06:49:03.314000,GGPP
 4404,1,2022-12-03 06:48:53.295000,GGPP
-5509188,1,2022-12-03 06:48:55.067000,GGPP
+5509188,1,2022-12-03 06:48:55.067000,GGPP
+3589760876641,1,2022-12-03 06:48:55.067000,GGPP
+3584045351010,1,2022-12-03 06:48:55.067000,GGPP

integration_tests/tests/consistency/consistency_customers.sql

@@ -0,0 +1,31 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_shopify_prod.shopify__customers
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_shopify_dev.shopify__customers
+), 
+
+final as (
+    -- test will fail if any rows from prod are not found in dev
+    (select * from prod
+    except distinct
+    select * from dev)
+
+    union all -- union since we only care if rows are produced
+
+    -- test will fail if any rows from dev are not found in prod
+    (select * from dev
+    except distinct
+    select * from prod)
+    )
+
+select *
+from final

integration_tests/tests/integrity/integrity_customers.sql

@@ -0,0 +1,35 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with source as (
+    select
+        customer_id,
+        count(*) as source_customer_tag_count
+    from {{ target.schema }}_shopify_dev.stg_shopify__customer_tag
+    group by 1
+),
+
+transform as (
+    select
+        customer_id,
+        array_length(split(customer_tags, ',')) as transform_customer_tag_count -- Only BigQuery compatible for the time being
+    from {{ target.schema }}_shopify_dev.shopify__customers
+    where customer_tags is not null
+    group by customer_id, customer_tags
+), 
+
+compare as (
+    select
+        source.customer_id,
+        source.source_customer_tag_count,
+        transform.transform_customer_tag_count
+    from source
+    full outer join transform
+    on source.customer_id = transform.customer_id
+    where source.source_customer_tag_count != transform.transform_customer_tag_count
+    )
+
+select *
+from compare

models/intermediate/shopify__customers__order_aggregates.sql

@@ -29,27 +29,11 @@ with orders as (
     from transactions
     {{ dbt_utils.group_by(n=3) }}
 
-), customer_tags as (
-
-    select *
-    from {{ var('shopify_customer_tag' )}}
-
-), customer_tags_aggregated as (
-
-    select 
-        customer_id,
-        source_relation,
-        {{ fivetran_utils.string_agg("distinct cast(value as " ~ dbt.type_string() ~ ")", "', '") }} as customer_tags
-
-    from customer_tags
-    group by 1,2
-
 ), aggregated as (
 
     select
         orders.customer_id,
         orders.source_relation,
-        customer_tags_aggregated.customer_tags,
         min(orders.created_timestamp) as first_order_timestamp,
         max(orders.created_timestamp) as most_recent_order_timestamp,
         avg(transaction_aggregates.currency_exchange_calculated_amount) as avg_order_value,
@@ -80,11 +64,8 @@ with orders as (
     left join order_aggregates
         on orders.order_id = order_aggregates.order_id
         and orders.source_relation = order_aggregates.source_relation
-    left join customer_tags_aggregated
-        on orders.customer_id = customer_tags_aggregated.customer_id
-        and orders.source_relation = customer_tags_aggregated.source_relation
 
-    {{ dbt_utils.group_by(n=3) }}
+    {{ dbt_utils.group_by(n=2) }}
 )
 
 select *

models/shopify__customers.sql

@@ -19,14 +19,24 @@ with customers as (
     where customer_id is not null
     group by 1,2
 
+), customer_tags_aggregated as (
+
+    select 
+        customer_id,
+        source_relation,
+        {{ fivetran_utils.string_agg("distinct cast(value as " ~ dbt.type_string() ~ ")", "', '") }} as customer_tags
+
+    from {{ var('shopify_customer_tag' )}}
+    group by 1,2
+
 ), joined as (
 
     select 
         customers.*,
         coalesce(abandoned.lifetime_abandoned_checkouts, 0) as lifetime_abandoned_checkouts,
         orders.first_order_timestamp,
         orders.most_recent_order_timestamp,
-        orders.customer_tags,
+        customer_tags_aggregated.customer_tags,
         orders.avg_order_value,
         coalesce(orders.lifetime_total_spent, 0) as lifetime_total_spent,
         coalesce(orders.lifetime_total_refunded, 0) as lifetime_total_refunded,
@@ -51,6 +61,9 @@ with customers as (
     left join abandoned
         on customers.customer_id = abandoned.customer_id
         and customers.source_relation = abandoned.source_relation
+    left join customer_tags_aggregated
+        on customers.customer_id = customer_tags_aggregated.customer_id
+        and customers.source_relation = customer_tags_aggregated.source_relation
 )
 
 select *