Bugfix/redshift constant expressions (#92)

* add abandoned_checkout config, update docs and versions * eventually change the deps * update docs, schema, script * updates * fix seed data identifier * add metadata config and update readme * update changelog * docs and revert project configs * changelog * update run script * new schema * small update to readme align * update readme * updates * Update CHANGELOG.md Co-authored-by: Joe Markiewicz <[email protected]> * format updates * update * package ref --------- Co-authored-by: Joe Markiewicz <[email protected]> Co-authored-by: Jamie Rodriguez <[email protected]>
fivetran · Dec 2, 2024 · 295a919 · 295a919
1 parent 903e562
commit 295a919
Show file tree

Hide file tree

Showing 24 changed files with 142 additions and 54 deletions.
diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh
@@ -21,15 +21,15 @@ if [ "$db" = "databricks-sql" ]; then
 dbt seed --vars '{shopify_schema: shopify_integrations_tests_sqlw}' --target "$db" --full-refresh
 dbt run --vars '{shopify_schema: shopify_integrations_tests_sqlw}' --target "$db" --full-refresh
 dbt test --vars '{shopify_schema: shopify_integrations_tests_sqlw}' --target "$db"
-dbt run --vars '{shopify_schema: shopify_integrations_tests_sqlw, shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_all_metafields: true, shopify__calendar_start_date: '2020-01-01'}' --target "$db" --full-refresh
+dbt run --vars '{shopify_schema: shopify_integrations_tests_sqlw, shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_all_metafields: true, shopify__calendar_start_date: '2020-01-01', shopify_using_abandoned_checkout: false, shopify_using_metafield: false}' --target "$db" --full-refresh
 dbt test --vars '{shopify_schema: shopify_integrations_tests_sqlw}' --target "$db"
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
 
 else
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_all_metafields: true, shopify__calendar_start_date: '2020-01-01'}' --target "$db" --full-refresh
+dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_all_metafields: true, shopify__calendar_start_date: '2020-01-01', shopify_using_abandoned_checkout: false, shopify_using_metafield: false}' --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
 fi
diff --git a/.quickstart/quickstart.yml b/.quickstart/quickstart.yml
@@ -4,8 +4,14 @@ schema_key: shopify_schema
 dbt_versions: ">=1.3.0 <2.0.0"
 
 table_variables:
+  shopify_using_metafield:
+    - metafield
   shopify_using_fulfillment_event:
     - fulfillment_event
+  shopify_using_abandoned_checkout:
+    - abandoned_checkout
+    - abandoned_checkout_discount_code
+    - abandoned_checkout_shipping_line
 
 destination_configurations:
   databricks:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,17 @@
+# dbt_shopify v0.14.0
+
+[PR #92](https://github.com/fivetran/dbt_shopify/pull/92) includes the following updates:
+## Breaking Changes
+- Adds enable/disable config for the `metadata` staging model using the `shopify_using_metafield` variable (default `true`).
+  - This variable is now a requirement for all `shopify__x_metafield` models.
+- Adds enable/disable config for the `abandoned_checkout` staging models using the `shopify_using_abandoned_checkout` variable (default `true`):
+   - `stg_shopify__abandoned_checkout`
+   - `stg_shopify__abandoned_checkout_discount_code`
+   - `stg_shopify__abandoned_checkout_shipping_line`.
+
+   - Disabling `shopify_using_abandoned_checkout` will also disable the `int_shopify__daily_abandoned_checkouts` and `int_shopify__discounts__abandoned_checkouts` intermediate models, in addition to disabling `abandoned_checkout` references in end models (including `shopify__daily_shop`, `shopify__customers`, `shopify__customer_emails`, `shopify__customer_email_cohorts`, `shopify__customer_cohorts`, and `shopify__discounts`).
+- For more information on how to enable/disable these tables, refer to the [README](https://github.com/fivetran/dbt_shopify/blob/main/README.md#step-4-disable-models-for-non-existent-sources). This will be a breaking change if you choose to disable these tables.
+
 # dbt_shopify v0.13.2
 [PR #89](https://github.com/fivetran/dbt_shopify/pull/89) includes the following changes:
 

diff --git a/README.md b/README.md
@@ -1,4 +1,6 @@
-<p align="center">
+# Shopify Transformation dbt Package ([Docs](https://fivetran.github.io/dbt_shopify/))
+
+<p align="left">
     <a alt="License"
         href="https://github.com/fivetran/dbt_shopify/blob/main/LICENSE">
         <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" /></a>
@@ -13,8 +15,6 @@
         <img src="https://img.shields.io/badge/Fivetran_Quickstart_Compatible%3F-yes-green.svg" /></a>
 </p>
 
-# Shopify Transformation dbt Package ([Docs](https://fivetran.github.io/dbt_shopify/))
-
 ## What does this dbt package do?
 
 This package models Shopify data from [Fivetran's connector](https://fivetran.com/docs/applications/shopify). It uses data in the format described by [this ERD](https://fivetran.com/docs/applications/shopify#schemainformation) and builds off the output of our [Shopify source package](https://github.com/fivetran/dbt_shopify_source).
@@ -70,7 +70,7 @@ If you are **not** using the [Shopify Holistic reporting package](https://github
 ```yml
 packages:
   - package: fivetran/shopify
-    version: [">=0.13.0", "<0.14.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.14.0", "<0.15.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 Do **NOT** include the `shopify_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well.
@@ -108,16 +108,17 @@ vars:
 
 To connect your multiple schema/database sources to the package models, follow the steps outlined in the [Union Data Defined Sources Configuration](https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source) section of the Fivetran Utils documentation for the union_data macro. This will ensure a proper configuration and correct visualization of connections in the DAG.
 
-### Step 4: Enable `fulfillment_event` data
+### Step 4: Disable models for non-existent sources
 
-The package takes into consideration that not every Shopify connector may have `fulfillment_event` data enabled. However, this table does hold valuable information that is leveraged in the `shopify__daily_shop` model. `fulfillment_event` data is **disabled by default**.
+The package takes into consideration that not every Shopify connector may have the `fulfillment_event`, `metadata`, or `abandoned_checkout` tables (including `abandoned_checkout`, `abandoned_checkout_discount_code`, and `abandoned_checkout_shipping_line`) and allows you to enable or disable the corresponding functionality. To enable/disable the modeling of the mentioned source tables and their downstream references, add the following variable to your `dbt_project.yml` file:
 
-Add the following variable to your `dbt_project.yml` file to enable the modeling of fulfillment events:
 ```yml
 # dbt_project.yml
 
 vars:
-    shopify_using_fulfillment_event: true # false by default
+    shopify_using_fulfillment_event: true # false by default. 
+    shopify_using_metafield: false  #true by default
+    shopify_using_abandoned_checkout: false # true by default. Setting to false will disable `abandoned_checkout`, `abandoned_checkout_discount_code`, and `abandoned_checkout_shipping_line`.
 ```
 
 ### Step 5: Setting your timezone
@@ -143,7 +144,7 @@ This package contains the `shopify__line_item_enhanced` model which constructs a
 vars:
   shopify__standardized_billing_model_enabled: true # false by default.
 ```
-    
+
 #### Passing Through Additional Fields
 This package includes all source columns defined in the macros folder. You can add more columns using our pass-through column variables. These variables allow for the pass-through fields to be aliased (`alias`) and casted (`transform_sql`) if desired, but not required. Datatype casting is configured via a sql snippet within the `transform_sql` key. You may add the desired sql while omitting the `as field_name` at the end and your custom pass-though fields will be casted accordingly. Use the below format for declaring the respective pass-through variables:
 
@@ -173,7 +174,10 @@ vars:
 
 #### Adding Metafields
 In [May 2021](https://fivetran.com/docs/applications/shopify/changelog#may2021) the Shopify connector included support for the [metafield resource](https://shopify.dev/api/admin-rest/2023-01/resources/metafield). If you would like to take advantage of these metafields, this package offers corresponding mapping models which append these metafields to the respective source object for the following tables: collection, customer, order, product_image, product, product_variant, shop. If enabled, these models will materialize as `shopify__[object]_metafields` for each respective supported object. To enable these metafield mapping models, you may use the following configurations within your `dbt_project.yml`.
->**Note**: These metafield models will contain all the same records as the corresponding staging models with the exception of the metafield columns being added.
+
+>**Note 1**: These metafield models will contain all the same records as the corresponding staging models with the exception of the metafield columns being added.
+
+>**Note 2**: Please ensure that the `shopify_using_metafield` is not disabled. (Enabled by default)
 
 ```yml
 vars:
@@ -250,7 +254,7 @@ This dbt package is dependent on the following dbt packages. These dependencies
 ```yml
 packages:
     - package: fivetran/shopify_source
-      version: [">=0.12.0", "<0.13.0"]
+      version: [">=0.13.0", "<0.14.0"]
 
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify'
-version: '0.13.2'
+version: '0.14.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
@@ -10,7 +10,7 @@ models:
       +materialized: ephemeral
       int_shopify__customer_email_rollup:
         +materialized: view # so we can use the dbt_utils.star macro
-        
+
 vars:
   shopify:
     shopify_customer: "{{ ref('stg_shopify__customer') }}"

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: shopify_integration_tests_11
+      schema: shopify_integration_tests_12
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: shopify_integration_tests_11
+      schema: shopify_integration_tests_12
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: shopify_integration_tests_11
+      schema: shopify_integration_tests_12
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: shopify_integration_tests_11
+      schema: shopify_integration_tests_12
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: shopify_integration_tests_11
+      schema: shopify_integration_tests_12
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'shopify_integration_tests'
-version: '0.13.2'
+version: '0.14.0'
 profile: 'integration_tests'
 config-version: 2
 
@@ -13,8 +13,8 @@ vars:
   # shopify_using_fulfillment_event: true
   # shopify_using_all_metafields: true
   # shopify__standardized_billing_model_enabled: true
+  shopify_schema: shopify_integration_tests_12
 
-  shopify_schema: shopify_integration_tests_11
   shopify_source:
     shopify_customer_identifier: "shopify_customer_data"
     shopify_order_line_refund_identifier: "shopify_order_line_refund_data"

diff --git a/models/intermediate/int_shopify__daily_abandoned_checkouts.sql b/models/intermediate/int_shopify__daily_abandoned_checkouts.sql
@@ -1,3 +1,5 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
+
 with abandoned_checkout as (
 
     select *

diff --git a/models/intermediate/int_shopify__discounts__abandoned_checkouts.sql b/models/intermediate/int_shopify__discounts__abandoned_checkouts.sql
@@ -1,3 +1,5 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
+
 with abandoned_checkout as (
 
     select *

diff --git a/models/metafields/shopify__collection_metafields.sql b/models/metafields/shopify__collection_metafields.sql
@@ -1,4 +1,4 @@
-{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_collection_metafields', False)) }}
+{{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_collection_metafields', False)) ) }}
 
 {{ get_metafields( 
     source_object = "stg_shopify__collection", 

diff --git a/models/metafields/shopify__customer_metafields.sql b/models/metafields/shopify__customer_metafields.sql
@@ -1,4 +1,4 @@
-{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_customer_metafields', False)) }}
+{{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_customer_metafields', False)) )}}
 
 {{ get_metafields( 
     source_object = "stg_shopify__customer", 

diff --git a/models/metafields/shopify__order_metafields.sql b/models/metafields/shopify__order_metafields.sql
@@ -1,4 +1,4 @@
-{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_order_metafields', False)) }}
+{{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_order_metafields', False)) ) }}
 
 {{ get_metafields( 
     source_object = "stg_shopify__order", 

diff --git a/models/metafields/shopify__product_image_metafields.sql b/models/metafields/shopify__product_image_metafields.sql
@@ -1,4 +1,4 @@
-{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_product_image_metafields', False)) }}
+{{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_product_image_metafields', False)) ) }}
 
 {{ get_metafields( 
     source_object = "stg_shopify__product_image", 

diff --git a/models/metafields/shopify__product_metafields.sql b/models/metafields/shopify__product_metafields.sql
@@ -1,4 +1,4 @@
-{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_product_metafields', False)) }}
+{{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_product_metafields', False)) ) }}
 
 {{ get_metafields( 
     source_object = "stg_shopify__product", 

diff --git a/models/metafields/shopify__product_variant_metafields.sql b/models/metafields/shopify__product_variant_metafields.sql
@@ -1,4 +1,4 @@
-{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_product_variant_metafields', False)) }}
+{{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_product_variant_metafields', False)) ) }}
 
 {{ get_metafields( 
     source_object = "stg_shopify__product_variant", 

diff --git a/models/metafields/shopify__shop_metafields.sql b/models/metafields/shopify__shop_metafields.sql
@@ -1,4 +1,4 @@
-{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_shop_metafields', False)) }}
+{{ config(enabled=var('shopify_using_metafield', True) and (var('shopify_using_all_metafields', False) or var('shopify_using_shop_metafields', False)) ) }}
 
 {{ get_metafields( 
     source_object = "stg_shopify__shop", 

diff --git a/models/shopify__customer_emails.sql b/models/shopify__customer_emails.sql
@@ -10,6 +10,7 @@ with customer_emails as (
     from {{ ref('int_shopify__emails__order_aggregates' )}}
     where email is not null
 
+{% if var('shopify_using_abandoned_checkout', True) %}
 ), abandoned as (
 
     select 
@@ -19,12 +20,17 @@ with customer_emails as (
     from {{ var('shopify_abandoned_checkout' )}}
     where email is not null
     group by 1,2
+{% endif %}
 
 ), joined as (
 
     select 
         customer_emails.*,
+
+        {% if var('shopify_using_abandoned_checkout', True) %}
         coalesce(abandoned.lifetime_abandoned_checkouts, 0) as lifetime_abandoned_checkouts,
+        {% endif %}
+
         orders.first_order_timestamp,
         orders.most_recent_order_timestamp,
         orders.avg_order_value,
@@ -48,9 +54,12 @@ with customer_emails as (
     left join orders
         on customer_emails.email = orders.email
         and customer_emails.source_relation = orders.source_relation
+
+    {% if var('shopify_using_abandoned_checkout', True) %}
     left join abandoned
         on customer_emails.email = abandoned.email
         and customer_emails.source_relation = abandoned.source_relation
+    {% endif %}
 )
 
 select *

diff --git a/models/shopify__customers.sql b/models/shopify__customers.sql
@@ -9,6 +9,7 @@ with customers as (
     select *
     from {{ ref('shopify__customers__order_aggregates' )}}
 
+{% if var('shopify_using_abandoned_checkout', True) %}
 ), abandoned as (
 
     select 
@@ -18,6 +19,7 @@ with customers as (
     from {{ var('shopify_abandoned_checkout' )}}
     where customer_id is not null
     group by 1,2
+{% endif %}
 
 ), customer_tags_aggregated as (
 
@@ -33,7 +35,11 @@ with customers as (
 
     select 
         customers.*,
+
+        {% if var('shopify_using_abandoned_checkout', True) %}
         coalesce(abandoned.lifetime_abandoned_checkouts, 0) as lifetime_abandoned_checkouts,
+        {% endif %}
+
         orders.first_order_timestamp,
         orders.most_recent_order_timestamp,
         customer_tags_aggregated.customer_tags,
@@ -58,12 +64,16 @@ with customers as (
     left join orders
         on customers.customer_id = orders.customer_id
         and customers.source_relation = orders.source_relation
-    left join abandoned
-        on customers.customer_id = abandoned.customer_id
-        and customers.source_relation = abandoned.source_relation
     left join customer_tags_aggregated
         on customers.customer_id = customer_tags_aggregated.customer_id
         and customers.source_relation = customer_tags_aggregated.source_relation
+
+    {% if var('shopify_using_abandoned_checkout', True) %}
+    left join abandoned
+        on customers.customer_id = abandoned.customer_id
+        and customers.source_relation = abandoned.source_relation
+    {% endif %}
+
 )
 
 select *

diff --git a/models/shopify__daily_shop.sql b/models/shopify__daily_shop.sql
@@ -17,11 +17,13 @@ daily_orders as (
     from {{ ref('int_shopify__daily_orders') }}
 ),
 
+{% if var('shopify_using_abandoned_checkout', True) %}
 daily_abandoned_checkouts as (
 
     select *
     from {{ ref('int_shopify__daily_abandoned_checkouts') }}
 ),
+{% endif %}
 
 {% if var('shopify_using_fulfillment_event', false) %}
 daily_fulfillment as (
@@ -81,7 +83,6 @@ final as (
         coalesce(daily_orders.count_orders_with_refunds, 0) as count_orders_with_refunds,
         daily_orders.first_order_timestamp,
         daily_orders.last_order_timestamp,
-
         coalesce(daily_orders.quantity_sold, 0) as quantity_sold,
         coalesce(daily_orders.quantity_refunded, 0) as quantity_refunded,
         coalesce(daily_orders.quantity_net, 0) as quantity_net,
@@ -90,11 +91,13 @@ final as (
         coalesce(daily_orders.count_variants_sold, 0) as count_variants_sold,
         coalesce(daily_orders.count_products_sold, 0) as count_products_sold,
         coalesce(daily_orders.quantity_gift_cards_sold, 0) as quantity_gift_cards_sold,
-        coalesce(daily_orders.quantity_requiring_shipping, 0) as quantity_requiring_shipping,
+        coalesce(daily_orders.quantity_requiring_shipping, 0) as quantity_requiring_shipping
 
-        coalesce(daily_abandoned_checkouts.count_abandoned_checkouts, 0) as count_abandoned_checkouts,
+        {% if var('shopify_using_abandoned_checkout', True) %}
+        , coalesce(daily_abandoned_checkouts.count_abandoned_checkouts, 0) as count_abandoned_checkouts,
         coalesce(daily_abandoned_checkouts.count_customers_abandoned_checkout, 0) as count_customers_abandoned_checkout,
         coalesce(daily_abandoned_checkouts.count_customer_emails_abandoned_checkout, 0) as count_customer_emails_abandoned_checkout
+        {% endif %}
 
         {% if var('shopify_using_fulfillment_event', false) %}
             {% for status in ['attempted_delivery', 'delayed', 'delivered', 'failure', 'in_transit', 'out_for_delivery', 'ready_for_pickup', 'picked_up', 'label_printed', 'label_purchased', 'confirmed']%}
@@ -106,9 +109,13 @@ final as (
     left join daily_orders 
         on shop_calendar.source_relation = daily_orders.source_relation
         and shop_calendar.date_day = daily_orders.date_day
+
+    {% if var('shopify_using_abandoned_checkout', True) %}
     left join daily_abandoned_checkouts 
         on shop_calendar.source_relation = daily_abandoned_checkouts.source_relation
         and shop_calendar.date_day = daily_abandoned_checkouts.date_day
+    {% endif %}
+
     {% if var('shopify_using_fulfillment_event', false) %}
     left join daily_fulfillment 
         on shop_calendar.source_relation = daily_fulfillment.source_relation