From 95f29598aa1b6a6132fedf89905522551e2a3f4c Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 09:41:20 -0400 Subject: [PATCH 01/23] add constraints to get_column_values macro --- macros/unpack/get_column_values.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/macros/unpack/get_column_values.sql b/macros/unpack/get_column_values.sql index d1607b57..cffba28e 100644 --- a/macros/unpack/get_column_values.sql +++ b/macros/unpack/get_column_values.sql @@ -24,6 +24,7 @@ wrap_string_with_quotes(dbt.escape_single_quotes(column.name)), wrap_string_with_quotes(dbt.escape_single_quotes(column.description)), wrap_string_with_quotes(dbt.escape_single_quotes(column.data_type)), + wrap_string_with_quotes(dbt.escape_single_quotes(tojson(column.constraints))), wrap_string_with_quotes(dbt.escape_single_quotes(column.quote)) ] %} From 74ca45e3eb79a29709c95ce8a1bc6f8b3b39edfb Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 09:42:14 -0400 Subject: [PATCH 02/23] add constraints to base_node_columns model --- models/staging/graph/base/base_node_columns.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/staging/graph/base/base_node_columns.sql b/models/staging/graph/base/base_node_columns.sql index 81dd5a12..99bece41 100644 --- a/models/staging/graph/base/base_node_columns.sql +++ b/models/staging/graph/base/base_node_columns.sql @@ -18,7 +18,8 @@ select cast(null as {{ dbt.type_string()}}) as name, cast(null as {{ dbt_project_evaluator.type_large_string()}}) as description, cast(null as {{ dbt.type_string()}}) as data_type, + cast(null as {{ dbt.type_string()}}) as constraints, cast(null as {{ dbt.type_string()}}) as quote from dummy_cte -where false \ No newline at end of file +where false From c6b09af499868a0efc3c6527ff0edd7ea964fccd Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 11:10:45 -0400 Subject: [PATCH 03/23] add constraints to int_model_test_summary model --- .../intermediate/int_model_test_summary.sql | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/models/marts/tests/intermediate/int_model_test_summary.sql b/models/marts/tests/intermediate/int_model_test_summary.sql index 62b432b8..1ab0f546 100644 --- a/models/marts/tests/intermediate/int_model_test_summary.sql +++ b/models/marts/tests/intermediate/int_model_test_summary.sql @@ -14,6 +14,7 @@ count_column_tests as ( select relationships.direct_parent_id, all_graph_resources.column_name, + if(max(all_graph_resources.is_test_unique), 1, 0) as test_unique_count, {%- for test_set in var('primary_key_test_macros') %} {%- set outer_loop = loop -%} count(distinct case when @@ -32,6 +33,29 @@ count_column_tests as ( group by 1,2 ), +count_column_constraints as ( + + select + node_unique_id as direct_parent_id, + name as column_name, + if(contains_substr(constraints, 'not_null'), 1, 0) as constraint_not_null_count, + cast((length(constraints) - length(replace(constraints, 'type', ''))) / 4 as int64) as constraints_count + from {{ ref('base_node_columns') }} + +), + +combine_column_counts as ( + + select + count_column_tests.*, + count_column_tests.test_unique_count + count_column_constraints.constraint_not_null_count as primary_key_mixed_method_count, + count_column_constraints.constraints_count + from count_column_tests + left join count_column_constraints + using (direct_parent_id, column_name) + +), + agg_test_relationships as ( select @@ -41,14 +65,16 @@ agg_test_relationships as ( {%- for test_set in var('primary_key_test_macros') %} {%- set compare_value = test_set | length %} primary_key_method_{{ loop.index }}_count >= {{ compare_value}} - {%- if not loop.last %} or {% endif %} - {%- endfor %} + or + {%- endfor %} + primary_key_mixed_method_count >= 2 ) then 1 else 0 end ) >= 1 as is_primary_key_tested, - sum(tests_count) as number_of_tests_on_model - from count_column_tests + sum(tests_count) as number_of_tests_on_model, + sum(constraints_count) as number_of_constraints_on_model + from combine_column_counts group by 1 ), @@ -59,7 +85,8 @@ final as ( all_graph_resources.resource_type, all_graph_resources.model_type, coalesce(agg_test_relationships.is_primary_key_tested, FALSE) as is_primary_key_tested, - coalesce(agg_test_relationships.number_of_tests_on_model, 0) as number_of_tests_on_model + coalesce(agg_test_relationships.number_of_tests_on_model, 0) as number_of_tests_on_model, + coalesce(agg_test_relationships.number_of_constraints_on_model, 0) as number_of_constraints_on_model from all_graph_resources left join agg_test_relationships on all_graph_resources.resource_id = agg_test_relationships.direct_parent_id From c9558280a17661084ca871b9fc14508395c4cbe8 Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 11:30:10 -0400 Subject: [PATCH 04/23] update fct_missing_primary_key_tests rule documentation --- docs/rules/testing.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/rules/testing.md b/docs/rules/testing.md index e1c1b654..3c27e894 100644 --- a/docs/rules/testing.md +++ b/docs/rules/testing.md @@ -6,7 +6,8 @@ `fct_missing_primary_key_tests` ([source](https://github.com/dbt-labs/dbt-project-evaluator/tree/main/models/marts/tests/fct_missing_primary_key_tests.sql)) lists every model that does not meet the minimum testing requirement of testing primary keys. Any model that does not have either 1. a `not_null` test and a `unique` test applied to a single column OR -2. a `dbt_utils.unique_combination_of_columns` test applied to a set of columns +2. a `dbt_utils.unique_combination_of_columns` test applied to a set of columns OR +3. a `not_null` constraint and a `unique` test applied to a single column will be flagged by this model. @@ -16,7 +17,7 @@ Tests are assertions you make about your models and other resources in your dbt **How to Remediate** -Apply a [uniqueness test](https://docs.getdbt.com/reference/resource-properties/tests#unique) and a [not null test](https://docs.getdbt.com/reference/resource-properties/tests#not_null) to the column that represents the grain of your model in its schema entry. For models that are unique across a combination of columns, we recommend adding a surrogate key column to your model, then applying these tests to that new model. See the [`surrogate_key`](https://github.com/dbt-labs/dbt-utils#surrogate_key-source) macro from dbt_utils for more info! Alternatively, you can use the [`dbt_utils.unique_combination_of_columns`](https://github.com/dbt-labs/dbt-utils#unique_combination_of_columns-source) test from `dbt_utils`. Check out the [overriding variables section](../customization/overriding-variables.md) to read more about configuring other primary key tests for your project! +Apply a [uniqueness test](https://docs.getdbt.com/reference/resource-properties/tests#unique) and a [not null test](https://docs.getdbt.com/reference/resource-properties/tests#not_null) to the column that represents the grain of your model in its schema entry. For contracted models, optionally replace the not null test with the not null [constraint](https://docs.getdbt.com/reference/resource-properties/constraints). For models that are unique across a combination of columns, we recommend adding a surrogate key column to your model, then applying these tests to that new model. See the [`surrogate_key`](https://github.com/dbt-labs/dbt-utils#surrogate_key-source) macro from dbt_utils for more info! Alternatively, you can use the [`dbt_utils.unique_combination_of_columns`](https://github.com/dbt-labs/dbt-utils#unique_combination_of_columns-source) test from `dbt_utils`. Check out the [overriding variables section](../customization/overriding-variables.md) to read more about configuring other primary key tests for your project! Additional tests can be configured by applying a [generic test](https://docs.getdbt.com/docs/building-a-dbt-project/tests#generic-tests) in the model's `.yml` entry or by creating a [singular test](https://docs.getdbt.com/docs/building-a-dbt-project/tests#singular-tests) in the `tests` directory of you project. From 523573e0a01ebd389b4cb5a1c8557be784eb07d2 Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 11:55:32 -0400 Subject: [PATCH 05/23] try updating fct_missing_primary_key_tests integration test --- .../models/marts/_int_model_5.yml | 14 +++++++++- .../test_fct_missing_primary_key_tests.csv | 28 +++++++++---------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/integration_tests/models/marts/_int_model_5.yml b/integration_tests/models/marts/_int_model_5.yml index 4b20b28b..7f93b57d 100644 --- a/integration_tests/models/marts/_int_model_5.yml +++ b/integration_tests/models/marts/_int_model_5.yml @@ -1,9 +1,21 @@ models: - name: int_model_5 latest_version: 2 + config: + materialized: table + contract: {enforced: true} + columns: + - name: id + data_type: string + description: "The primary key for this table" + constraints: + - type: not_null + tests: + - unique + versions: - v: 1 deprecation_date: 2024-01-01 - v: 2 deprecation_date: 2029-01-01 - - v: 3 \ No newline at end of file + - v: 3 diff --git a/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv b/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv index 46204a98..6315992f 100644 --- a/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv +++ b/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv @@ -1,14 +1,14 @@ -resource_name,is_primary_key_tested,number_of_tests_on_model -dim_model_7,FALSE,0 -fct_model_6,FALSE,0 -model_8,FALSE,0 -fct_model_9,FALSE,0 -int_model_5.v1,FALSE,0 -int_model_5.v2,FALSE,0 -int_model_5.v3,FALSE,0 -report_1,FALSE,0 -report_2,FALSE,0 -report_3,FALSE,0 -stg_model_1,FALSE,2 -stg_model_5,FALSE,0 -fct_model_10,FALSE,0 +resource_name,is_primary_key_tested,number_of_tests_on_model,number_of_constraints_on_model +dim_model_7,FALSE,0,0 +fct_model_6,FALSE,0,0 +model_8,FALSE,0,0 +fct_model_9,FALSE,0,0 +int_model_5.v1,TRUE,1,1 +int_model_5.v2,TRUE,1,1 +int_model_5.v3,TRUE,1,1 +report_1,FALSE,0,0 +report_2,FALSE,0,0 +report_3,FALSE,0,0 +stg_model_1,FALSE,2,0 +stg_model_5,FALSE,0,0 +fct_model_10,FALSE,0,0 From 7965b231aebdfaed70554f933921e8678e63d8ec Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 12:55:45 -0400 Subject: [PATCH 06/23] fix contracted column data type --- integration_tests/models/marts/_int_model_5.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/models/marts/_int_model_5.yml b/integration_tests/models/marts/_int_model_5.yml index 7f93b57d..bd123120 100644 --- a/integration_tests/models/marts/_int_model_5.yml +++ b/integration_tests/models/marts/_int_model_5.yml @@ -6,7 +6,7 @@ models: contract: {enforced: true} columns: - name: id - data_type: string + data_type: int64 description: "The primary key for this table" constraints: - type: not_null From d6aec48b8892d8519dace3b0874afb27b3863b6e Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 13:00:02 -0400 Subject: [PATCH 07/23] update test coverage integration test seed --- integration_tests/seeds/tests/test_fct_test_coverage.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/seeds/tests/test_fct_test_coverage.csv b/integration_tests/seeds/tests/test_fct_test_coverage.csv index 7f2a65fb..1c541e9c 100644 --- a/integration_tests/seeds/tests/test_fct_test_coverage.csv +++ b/integration_tests/seeds/tests/test_fct_test_coverage.csv @@ -1,2 +1,2 @@ total_models,total_tests,tested_models,test_coverage_pct,staging_test_coverage_pct,intermediate_test_coverage_pct,marts_test_coverage_pct,other_test_coverage_pct,test_to_model_ratio -18,13,6,33.33,80.00,25.00,0.00,25.00,0.7222 +18,16,9,50.00,80.00,100.0,0.00,25.00,0.8889 From 9f791b0d7d3a33a5098bb1321cbc211fda75feae Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 13:40:00 -0400 Subject: [PATCH 08/23] revert changes to int_model_5 --- .../models/marts/_int_model_5.yml | 14 +--------- .../test_fct_missing_primary_key_tests.csv | 28 +++++++++---------- .../seeds/tests/test_fct_test_coverage.csv | 2 +- 3 files changed, 16 insertions(+), 28 deletions(-) diff --git a/integration_tests/models/marts/_int_model_5.yml b/integration_tests/models/marts/_int_model_5.yml index bd123120..4b20b28b 100644 --- a/integration_tests/models/marts/_int_model_5.yml +++ b/integration_tests/models/marts/_int_model_5.yml @@ -1,21 +1,9 @@ models: - name: int_model_5 latest_version: 2 - config: - materialized: table - contract: {enforced: true} - columns: - - name: id - data_type: int64 - description: "The primary key for this table" - constraints: - - type: not_null - tests: - - unique - versions: - v: 1 deprecation_date: 2024-01-01 - v: 2 deprecation_date: 2029-01-01 - - v: 3 + - v: 3 \ No newline at end of file diff --git a/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv b/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv index 6315992f..46204a98 100644 --- a/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv +++ b/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv @@ -1,14 +1,14 @@ -resource_name,is_primary_key_tested,number_of_tests_on_model,number_of_constraints_on_model -dim_model_7,FALSE,0,0 -fct_model_6,FALSE,0,0 -model_8,FALSE,0,0 -fct_model_9,FALSE,0,0 -int_model_5.v1,TRUE,1,1 -int_model_5.v2,TRUE,1,1 -int_model_5.v3,TRUE,1,1 -report_1,FALSE,0,0 -report_2,FALSE,0,0 -report_3,FALSE,0,0 -stg_model_1,FALSE,2,0 -stg_model_5,FALSE,0,0 -fct_model_10,FALSE,0,0 +resource_name,is_primary_key_tested,number_of_tests_on_model +dim_model_7,FALSE,0 +fct_model_6,FALSE,0 +model_8,FALSE,0 +fct_model_9,FALSE,0 +int_model_5.v1,FALSE,0 +int_model_5.v2,FALSE,0 +int_model_5.v3,FALSE,0 +report_1,FALSE,0 +report_2,FALSE,0 +report_3,FALSE,0 +stg_model_1,FALSE,2 +stg_model_5,FALSE,0 +fct_model_10,FALSE,0 diff --git a/integration_tests/seeds/tests/test_fct_test_coverage.csv b/integration_tests/seeds/tests/test_fct_test_coverage.csv index 1c541e9c..7f2a65fb 100644 --- a/integration_tests/seeds/tests/test_fct_test_coverage.csv +++ b/integration_tests/seeds/tests/test_fct_test_coverage.csv @@ -1,2 +1,2 @@ total_models,total_tests,tested_models,test_coverage_pct,staging_test_coverage_pct,intermediate_test_coverage_pct,marts_test_coverage_pct,other_test_coverage_pct,test_to_model_ratio -18,16,9,50.00,80.00,100.0,0.00,25.00,0.8889 +18,13,6,33.33,80.00,25.00,0.00,25.00,0.7222 From bb59c5484007ddb1598b919ebd9e8c99dd0af807 Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 13:48:50 -0400 Subject: [PATCH 09/23] add primary key test and constraint to dim_model_7 --- .../models/marts/intermediate/_dim_model_7.yml | 12 ++++++++++++ .../tests/test_fct_missing_primary_key_tests.csv | 1 - .../seeds/tests/test_fct_test_coverage.csv | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 integration_tests/models/marts/intermediate/_dim_model_7.yml diff --git a/integration_tests/models/marts/intermediate/_dim_model_7.yml b/integration_tests/models/marts/intermediate/_dim_model_7.yml new file mode 100644 index 00000000..53a11444 --- /dev/null +++ b/integration_tests/models/marts/intermediate/_dim_model_7.yml @@ -0,0 +1,12 @@ +models: + - name: dim_model_7 + config: + contract: + enforced: true + columns: + - name: id + data_type: int64 + constraints: + - type: not_null + tests: + - unique diff --git a/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv b/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv index 46204a98..4eacdd1c 100644 --- a/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv +++ b/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv @@ -1,5 +1,4 @@ resource_name,is_primary_key_tested,number_of_tests_on_model -dim_model_7,FALSE,0 fct_model_6,FALSE,0 model_8,FALSE,0 fct_model_9,FALSE,0 diff --git a/integration_tests/seeds/tests/test_fct_test_coverage.csv b/integration_tests/seeds/tests/test_fct_test_coverage.csv index 7f2a65fb..16546318 100644 --- a/integration_tests/seeds/tests/test_fct_test_coverage.csv +++ b/integration_tests/seeds/tests/test_fct_test_coverage.csv @@ -1,2 +1,2 @@ total_models,total_tests,tested_models,test_coverage_pct,staging_test_coverage_pct,intermediate_test_coverage_pct,marts_test_coverage_pct,other_test_coverage_pct,test_to_model_ratio -18,13,6,33.33,80.00,25.00,0.00,25.00,0.7222 +18,14,7,38.89,80.00,25.00,20.00,25.00,0.7778 From b1caa752f41e0eb275790eefcc26e8e6049c281a Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 13:54:17 -0400 Subject: [PATCH 10/23] try int --- integration_tests/models/marts/intermediate/_dim_model_7.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/models/marts/intermediate/_dim_model_7.yml b/integration_tests/models/marts/intermediate/_dim_model_7.yml index 53a11444..6707dd10 100644 --- a/integration_tests/models/marts/intermediate/_dim_model_7.yml +++ b/integration_tests/models/marts/intermediate/_dim_model_7.yml @@ -5,7 +5,7 @@ models: enforced: true columns: - name: id - data_type: int64 + data_type: int constraints: - type: not_null tests: From 507b3ca2d629f2f194a6385b2d3f3b9023bd506a Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 14:20:29 -0400 Subject: [PATCH 11/23] use cross db bool to text --- models/marts/tests/intermediate/int_model_test_summary.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/tests/intermediate/int_model_test_summary.sql b/models/marts/tests/intermediate/int_model_test_summary.sql index 1ab0f546..e633c26a 100644 --- a/models/marts/tests/intermediate/int_model_test_summary.sql +++ b/models/marts/tests/intermediate/int_model_test_summary.sql @@ -14,7 +14,7 @@ count_column_tests as ( select relationships.direct_parent_id, all_graph_resources.column_name, - if(max(all_graph_resources.is_test_unique), 1, 0) as test_unique_count, + if(max({{ dbt.cast_bool_to_text("all_graph_resources.is_test_unique") }}) = 'true', 1, 0) as test_unique_count, {%- for test_set in var('primary_key_test_macros') %} {%- set outer_loop = loop -%} count(distinct case when From 34bdd1fca81780cf6f9a5c99f5e8a81f8d4cee5b Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 14:30:27 -0400 Subject: [PATCH 12/23] replace if function with summed case statement --- models/marts/tests/intermediate/int_model_test_summary.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/models/marts/tests/intermediate/int_model_test_summary.sql b/models/marts/tests/intermediate/int_model_test_summary.sql index e633c26a..5b36b90b 100644 --- a/models/marts/tests/intermediate/int_model_test_summary.sql +++ b/models/marts/tests/intermediate/int_model_test_summary.sql @@ -14,7 +14,12 @@ count_column_tests as ( select relationships.direct_parent_id, all_graph_resources.column_name, - if(max({{ dbt.cast_bool_to_text("all_graph_resources.is_test_unique") }}) = 'true', 1, 0) as test_unique_count, + sum(case + when all_graph_resources.is_test_unique + then 1 + else 0 + end + ) as test_unique_count, {%- for test_set in var('primary_key_test_macros') %} {%- set outer_loop = loop -%} count(distinct case when From 20f9125832a09bdcd470ee09145bb962acdd2e82 Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 17:29:38 -0400 Subject: [PATCH 13/23] use cross db replace function to flag not null constraint --- models/marts/tests/intermediate/int_model_test_summary.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/models/marts/tests/intermediate/int_model_test_summary.sql b/models/marts/tests/intermediate/int_model_test_summary.sql index 5b36b90b..0c41c1d0 100644 --- a/models/marts/tests/intermediate/int_model_test_summary.sql +++ b/models/marts/tests/intermediate/int_model_test_summary.sql @@ -43,7 +43,11 @@ count_column_constraints as ( select node_unique_id as direct_parent_id, name as column_name, - if(contains_substr(constraints, 'not_null'), 1, 0) as constraint_not_null_count, + case + when {{ dbt.replace("constraints", "not_null", "") }} != constraints + then 1 + else 0 + end as constraint_not_null_count, cast((length(constraints) - length(replace(constraints, 'type', ''))) / 4 as int64) as constraints_count from {{ ref('base_node_columns') }} From 4ddd51b2b810b9f91533f7aebc6ab82c9d10021b Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 17:34:43 -0400 Subject: [PATCH 14/23] add inner single quotes --- models/marts/tests/intermediate/int_model_test_summary.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/tests/intermediate/int_model_test_summary.sql b/models/marts/tests/intermediate/int_model_test_summary.sql index 0c41c1d0..c4e6ed07 100644 --- a/models/marts/tests/intermediate/int_model_test_summary.sql +++ b/models/marts/tests/intermediate/int_model_test_summary.sql @@ -44,7 +44,7 @@ count_column_constraints as ( node_unique_id as direct_parent_id, name as column_name, case - when {{ dbt.replace("constraints", "not_null", "") }} != constraints + when {{ replace("constraints", "'not_null'", "''") }} != constraints then 1 else 0 end as constraint_not_null_count, From 1988b3dc0390fd4fa3cfdd8032928e2179896e28 Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 17:36:53 -0400 Subject: [PATCH 15/23] try int --- models/marts/tests/intermediate/int_model_test_summary.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/tests/intermediate/int_model_test_summary.sql b/models/marts/tests/intermediate/int_model_test_summary.sql index c4e6ed07..1e3cf411 100644 --- a/models/marts/tests/intermediate/int_model_test_summary.sql +++ b/models/marts/tests/intermediate/int_model_test_summary.sql @@ -48,7 +48,7 @@ count_column_constraints as ( then 1 else 0 end as constraint_not_null_count, - cast((length(constraints) - length(replace(constraints, 'type', ''))) / 4 as int64) as constraints_count + cast((length(constraints) - length(replace(constraints, 'type', ''))) / 4 as int) as constraints_count from {{ ref('base_node_columns') }} ), From 5445a5d76d23e9d278c1aab1dc68a3cf8c7515a4 Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 17:49:17 -0400 Subject: [PATCH 16/23] use simple replace --- models/marts/tests/intermediate/int_model_test_summary.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/tests/intermediate/int_model_test_summary.sql b/models/marts/tests/intermediate/int_model_test_summary.sql index 1e3cf411..25fd267b 100644 --- a/models/marts/tests/intermediate/int_model_test_summary.sql +++ b/models/marts/tests/intermediate/int_model_test_summary.sql @@ -44,7 +44,7 @@ count_column_constraints as ( node_unique_id as direct_parent_id, name as column_name, case - when {{ replace("constraints", "'not_null'", "''") }} != constraints + when replace("constraints", "'not_null'", "''") != constraints then 1 else 0 end as constraint_not_null_count, From da3a72c835cf101550106db5c9f4e4ccd96958c7 Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Fri, 10 May 2024 17:50:00 -0400 Subject: [PATCH 17/23] remove double quotes --- models/marts/tests/intermediate/int_model_test_summary.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/tests/intermediate/int_model_test_summary.sql b/models/marts/tests/intermediate/int_model_test_summary.sql index 25fd267b..34703ad5 100644 --- a/models/marts/tests/intermediate/int_model_test_summary.sql +++ b/models/marts/tests/intermediate/int_model_test_summary.sql @@ -44,7 +44,7 @@ count_column_constraints as ( node_unique_id as direct_parent_id, name as column_name, case - when replace("constraints", "'not_null'", "''") != constraints + when replace(constraints, 'not_null', '') != constraints then 1 else 0 end as constraint_not_null_count, From e66a238e625c91aae8f10e75fb9809b16401b06e Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Sun, 12 May 2024 11:38:43 -0400 Subject: [PATCH 18/23] replace using syntax --- models/marts/tests/intermediate/int_model_test_summary.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/marts/tests/intermediate/int_model_test_summary.sql b/models/marts/tests/intermediate/int_model_test_summary.sql index 34703ad5..d701a77e 100644 --- a/models/marts/tests/intermediate/int_model_test_summary.sql +++ b/models/marts/tests/intermediate/int_model_test_summary.sql @@ -61,7 +61,8 @@ combine_column_counts as ( count_column_constraints.constraints_count from count_column_tests left join count_column_constraints - using (direct_parent_id, column_name) + on count_column_tests.direct_parent_id = count_column_constraints.direct_parent_id + and count_column_tests.column_name = count_column_constraints.column_name ), From e72717595511b213304b248f1a94516a2c9740d3 Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Sun, 12 May 2024 11:53:07 -0400 Subject: [PATCH 19/23] bump trino stage count --- integration_tests/ci/sample.profiles.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 183a3e4d..72aba86d 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -71,4 +71,4 @@ integration_tests: schema: dbt_project_evaluator_integration_tests_trino threads: 5 session_properties: - query_max_stage_count: 200 + query_max_stage_count: 250 From 1e78d975b492a8a08895bb4a090e8b42c12f692c Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Sun, 12 May 2024 12:10:31 -0400 Subject: [PATCH 20/23] bump trino stage count --- integration_tests/ci/sample.profiles.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 72aba86d..392271f3 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -71,4 +71,4 @@ integration_tests: schema: dbt_project_evaluator_integration_tests_trino threads: 5 session_properties: - query_max_stage_count: 250 + query_max_stage_count: 275 From aea85e1f7f49f370761cbebc0ca1aea4f782e2a5 Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Wed, 29 May 2024 05:43:27 -0400 Subject: [PATCH 21/23] search for exact not_null type --- models/marts/tests/intermediate/int_model_test_summary.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/tests/intermediate/int_model_test_summary.sql b/models/marts/tests/intermediate/int_model_test_summary.sql index d701a77e..6ed36087 100644 --- a/models/marts/tests/intermediate/int_model_test_summary.sql +++ b/models/marts/tests/intermediate/int_model_test_summary.sql @@ -44,7 +44,7 @@ count_column_constraints as ( node_unique_id as direct_parent_id, name as column_name, case - when replace(constraints, 'not_null', '') != constraints + when replace(constraints, '"not_null"', '') != constraints then 1 else 0 end as constraint_not_null_count, From ac8f2088f46b996c479d659990a593a2a03d9fe4 Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Mon, 10 Jun 2024 08:07:16 -0400 Subject: [PATCH 22/23] push logic back into base_node_columns, via get_column_values --- macros/unpack/get_column_values.sql | 2 ++ models/marts/tests/intermediate/int_model_test_summary.sql | 4 ++-- models/staging/graph/base/base_node_columns.sql | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/macros/unpack/get_column_values.sql b/macros/unpack/get_column_values.sql index cffba28e..5e1f5108 100644 --- a/macros/unpack/get_column_values.sql +++ b/macros/unpack/get_column_values.sql @@ -25,6 +25,8 @@ wrap_string_with_quotes(dbt.escape_single_quotes(column.description)), wrap_string_with_quotes(dbt.escape_single_quotes(column.data_type)), wrap_string_with_quotes(dbt.escape_single_quotes(tojson(column.constraints))), + column.constraints | selectattr('type', 'equalto', 'not_null') | list | length > 0, + column.constraints | length, wrap_string_with_quotes(dbt.escape_single_quotes(column.quote)) ] %} diff --git a/models/marts/tests/intermediate/int_model_test_summary.sql b/models/marts/tests/intermediate/int_model_test_summary.sql index 6ed36087..8cc64127 100644 --- a/models/marts/tests/intermediate/int_model_test_summary.sql +++ b/models/marts/tests/intermediate/int_model_test_summary.sql @@ -44,11 +44,11 @@ count_column_constraints as ( node_unique_id as direct_parent_id, name as column_name, case - when replace(constraints, '"not_null"', '') != constraints + when has_not_null_constraint then 1 else 0 end as constraint_not_null_count, - cast((length(constraints) - length(replace(constraints, 'type', ''))) / 4 as int) as constraints_count + constraints_count from {{ ref('base_node_columns') }} ), diff --git a/models/staging/graph/base/base_node_columns.sql b/models/staging/graph/base/base_node_columns.sql index 99bece41..d08954c1 100644 --- a/models/staging/graph/base/base_node_columns.sql +++ b/models/staging/graph/base/base_node_columns.sql @@ -19,6 +19,8 @@ select cast(null as {{ dbt_project_evaluator.type_large_string()}}) as description, cast(null as {{ dbt.type_string()}}) as data_type, cast(null as {{ dbt.type_string()}}) as constraints, + cast(True as boolean) as has_not_null_constraint, + cast(0 as {{ dbt.type_int() }}) as constraints_count, cast(null as {{ dbt.type_string()}}) as quote from dummy_cte From e23712dd527a42d8d73b903df46ef2b38f46ca4d Mon Sep 17 00:00:00 2001 From: Katie Claiborne Date: Tue, 11 Jun 2024 08:51:27 -0400 Subject: [PATCH 23/23] add number of contraints to test seed, update equality test based on dbt utils 1.2 release --- .../test_fct_missing_primary_key_tests.csv | 26 +++++++++---------- integration_tests/seeds/tests/tests_seeds.yml | 3 +++ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv b/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv index 4eacdd1c..d4d2e3da 100644 --- a/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv +++ b/integration_tests/seeds/tests/test_fct_missing_primary_key_tests.csv @@ -1,13 +1,13 @@ -resource_name,is_primary_key_tested,number_of_tests_on_model -fct_model_6,FALSE,0 -model_8,FALSE,0 -fct_model_9,FALSE,0 -int_model_5.v1,FALSE,0 -int_model_5.v2,FALSE,0 -int_model_5.v3,FALSE,0 -report_1,FALSE,0 -report_2,FALSE,0 -report_3,FALSE,0 -stg_model_1,FALSE,2 -stg_model_5,FALSE,0 -fct_model_10,FALSE,0 +resource_name,is_primary_key_tested,number_of_tests_on_model,number_of_constraints_on_model +fct_model_6,FALSE,0,0 +model_8,FALSE,0,0 +fct_model_9,FALSE,0,0 +int_model_5.v1,FALSE,0,0 +int_model_5.v2,FALSE,0,0 +int_model_5.v3,FALSE,0,0 +report_1,FALSE,0,0 +report_2,FALSE,0,0 +report_3,FALSE,0,0 +stg_model_1,FALSE,2,0 +stg_model_5,FALSE,0,0 +fct_model_10,FALSE,0,0 diff --git a/integration_tests/seeds/tests/tests_seeds.yml b/integration_tests/seeds/tests/tests_seeds.yml index de480a01..91aeabff 100644 --- a/integration_tests/seeds/tests/tests_seeds.yml +++ b/integration_tests/seeds/tests/tests_seeds.yml @@ -6,6 +6,9 @@ seeds: - dbt_utils.equality: name: equality_fct_missing_primary_key_tests compare_model: ref('fct_missing_primary_key_tests') + exclude_columns: + - resource_type + - model_type - name: test_fct_test_coverage config: