From 3c652c23ec6665aeff2b38d565cdccfdb31eaacc Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Mon, 14 Aug 2023 10:29:57 -0500 Subject: [PATCH 1/2] add excluded package --- integration_tests/dbt_project.yml | 2 +- .../exclude_package/dbt_project.yml | 30 +++++++++++++++++++ .../exclude_package/models/_models.yml | 11 +++++++ .../exclude_package/models/excluded_model.sql | 3 ++ integration_tests/packages.yml | 3 +- ..._exposures_dependent_on_private_models.sql | 1 + .../fct_public_models_without_contract.sql | 1 + .../fct_undocumented_public_models.sql | 1 + .../marts/structure/fct_model_directories.sql | 1 + 9 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 integration_tests/exclude_package/dbt_project.yml create mode 100644 integration_tests/exclude_package/models/_models.yml create mode 100644 integration_tests/exclude_package/models/excluded_model.sql diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index ca3934d5..02003e48 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -59,7 +59,7 @@ seeds: vars: # ensure integration tests run successfully when there are 0 of a given model type (extra) - exclude_packages: [] + exclude_packages: ['exclude_package'] exclude_paths_from_project: ["/to_exclude/","source_3.table_6"] model_types: ['staging', 'intermediate', 'marts', 'other', 'extra', 'new_model_type'] # dummy variable used for testing fct_hard_coded_references diff --git a/integration_tests/exclude_package/dbt_project.yml b/integration_tests/exclude_package/dbt_project.yml new file mode 100644 index 00000000..1dd54d6f --- /dev/null +++ b/integration_tests/exclude_package/dbt_project.yml @@ -0,0 +1,30 @@ + +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'exclude_package' +version: '1.0.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: 'integration_tests' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + +models: + exclude_package: + # materialize as ephemeral to prevent the fake models from executing, but keep them enabled + +materialized: view \ No newline at end of file diff --git a/integration_tests/exclude_package/models/_models.yml b/integration_tests/exclude_package/models/_models.yml new file mode 100644 index 00000000..2d71b738 --- /dev/null +++ b/integration_tests/exclude_package/models/_models.yml @@ -0,0 +1,11 @@ +models: + - name: excluded_model + access: public + config: + contract: + enforced: true + columns: + - name: id + data_type: integer + - name: data_quality + data_type: "{{ 'string' if target.type in ['bigquery', 'databricks'] else 'text' }}" \ No newline at end of file diff --git a/integration_tests/exclude_package/models/excluded_model.sql b/integration_tests/exclude_package/models/excluded_model.sql new file mode 100644 index 00000000..5af0ceda --- /dev/null +++ b/integration_tests/exclude_package/models/excluded_model.sql @@ -0,0 +1,3 @@ +select + 1 as id, + 'good' as data_quality \ No newline at end of file diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml index b16d3072..eb718713 100644 --- a/integration_tests/packages.yml +++ b/integration_tests/packages.yml @@ -1,2 +1,3 @@ packages: - - local: ../ \ No newline at end of file + - local: ../ + - local: exclude_package/ \ No newline at end of file diff --git a/models/marts/governance/fct_exposures_dependent_on_private_models.sql b/models/marts/governance/fct_exposures_dependent_on_private_models.sql index 65e9171c..6296eb6d 100644 --- a/models/marts/governance/fct_exposures_dependent_on_private_models.sql +++ b/models/marts/governance/fct_exposures_dependent_on_private_models.sql @@ -9,6 +9,7 @@ direct_exposure_relationships as ( parent_resource_type = 'model' and parent_is_public ) + and not parent_is_excluded ), final as ( diff --git a/models/marts/governance/fct_public_models_without_contract.sql b/models/marts/governance/fct_public_models_without_contract.sql index 39783a91..66524302 100644 --- a/models/marts/governance/fct_public_models_without_contract.sql +++ b/models/marts/governance/fct_public_models_without_contract.sql @@ -2,6 +2,7 @@ with all_resources as ( select * from {{ ref('int_all_graph_resources') }} + where not is_excluded ), final as ( diff --git a/models/marts/governance/fct_undocumented_public_models.sql b/models/marts/governance/fct_undocumented_public_models.sql index 491371d4..55da8e64 100644 --- a/models/marts/governance/fct_undocumented_public_models.sql +++ b/models/marts/governance/fct_undocumented_public_models.sql @@ -2,6 +2,7 @@ with all_resources as ( select * from {{ ref('int_all_graph_resources') }} + where not is_excluded ), final as ( diff --git a/models/marts/structure/fct_model_directories.sql b/models/marts/structure/fct_model_directories.sql index 1beb90db..a4feb756 100644 --- a/models/marts/structure/fct_model_directories.sql +++ b/models/marts/structure/fct_model_directories.sql @@ -5,6 +5,7 @@ with all_graph_resources as ( select * from {{ ref('int_all_graph_resources') }} + where not is_excluded ), folders as ( From b3b3a04862a7aa3b74e0411e8df2e5c8cc3a5cfc Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Mon, 14 Aug 2023 10:45:51 -0500 Subject: [PATCH 2/2] remove unecessary complexity --- integration_tests/exclude_package/models/_models.yml | 4 +--- integration_tests/exclude_package/models/excluded_model.sql | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/integration_tests/exclude_package/models/_models.yml b/integration_tests/exclude_package/models/_models.yml index 2d71b738..bbfb10d2 100644 --- a/integration_tests/exclude_package/models/_models.yml +++ b/integration_tests/exclude_package/models/_models.yml @@ -6,6 +6,4 @@ models: enforced: true columns: - name: id - data_type: integer - - name: data_quality - data_type: "{{ 'string' if target.type in ['bigquery', 'databricks'] else 'text' }}" \ No newline at end of file + data_type: integer \ No newline at end of file diff --git a/integration_tests/exclude_package/models/excluded_model.sql b/integration_tests/exclude_package/models/excluded_model.sql index 5af0ceda..26d9cae7 100644 --- a/integration_tests/exclude_package/models/excluded_model.sql +++ b/integration_tests/exclude_package/models/excluded_model.sql @@ -1,3 +1 @@ -select - 1 as id, - 'good' as data_quality \ No newline at end of file +select 1 as id \ No newline at end of file