From 0b62d64e9e8456a025055fe5f4577cd88c6913f9 Mon Sep 17 00:00:00 2001 From: z3z1ma Date: Tue, 31 Dec 2024 13:09:09 -0700 Subject: [PATCH] feat: be more liberal in accepting config keys of dbt_osmosis such as to support config block syntax --- src/dbt_osmosis/core/osmosis.py | 27 +++++++++++++------ .../test_column_level_knowledge_propagator.py | 5 ++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/dbt_osmosis/core/osmosis.py b/src/dbt_osmosis/core/osmosis.py index fed19bc..ba359c9 100644 --- a/src/dbt_osmosis/core/osmosis.py +++ b/src/dbt_osmosis/core/osmosis.py @@ -383,14 +383,20 @@ def mutated(self) -> bool: @property def source_definitions(self) -> dict[str, t.Any]: """The source definitions from the dbt project config.""" - defs = self.project.config.vars.to_dict().get("dbt-osmosis", {}).copy() + c = self.project.config.vars.to_dict() + defs = _find_first( + [c.get(k, {}) for k in ["dbt-osmosis", "dbt_osmosis"]], lambda v: bool(v), {} + ) defs.pop(SKIP_PATTERNS, None) return defs @property def skip_patterns(self) -> list[str]: """The column name skip patterns from the dbt project config.""" - defs = self.project.config.vars.to_dict().get("dbt-osmosis", {}).copy() + c = self.project.config.vars.to_dict() + defs = _find_first( + [c.get(k, {}) for k in ["dbt-osmosis", "dbt_osmosis"]], lambda v: bool(v), {} + ) return defs.pop(SKIP_PATTERNS, []) def read_catalog(self) -> CatalogArtifact | None: @@ -716,7 +722,12 @@ def _get_yaml_path_template(context: YamlRefactorContext, node: ResultNode) -> s if isinstance(def_or_path, dict): return def_or_path.get("path") return def_or_path - path_template = node.config.extra.get("dbt-osmosis", node.unrendered_config.get("dbt-osmosis")) + conf = [ + c.get(k) + for k in ["dbt-osmosis", "dbt_osmosis"] + for c in [node.config.extra, node.unrendered_config] + ] + path_template = _find_first(t.cast(list[str | None], conf), lambda v: v is not None) if not path_template: raise MissingOsmosisConfig( f"Config key `dbt-osmosis: ` not set for model {node.name}" @@ -1326,12 +1337,12 @@ def get_candidates(self, name: str, node: ResultNode, context: DbtProjectContext variants = [] prefix = t.cast( str, - node.config.extra.get( - "dbt-osmosis-prefix", node.unrendered_config.get("dbt-osmosis-prefix") - ), + node.meta.get("osmosis_prefix") # Can be set in the node yml (legacy support) + or node.config.extra.get("dbt_osmosis_prefix") # Or in dbt_project.yml / {{ config() }} + or node.unrendered_config.get("dbt_osmosis_prefix"), ) - if prefix and name.startswith(prefix): - variants.append(name[len(prefix) :]) + if prefix: + variants.append(name.removeprefix(prefix)) return variants diff --git a/tests/test_column_level_knowledge_propagator.py b/tests/test_column_level_knowledge_propagator.py index cc18c07..2af5312 100644 --- a/tests/test_column_level_knowledge_propagator.py +++ b/tests/test_column_level_knowledge_propagator.py @@ -441,6 +441,8 @@ def test_update_undocumented_columns_with_prior_knowledge_with_osmosis_keep_desc assert col_0["description"] == column_description_not_updated +# NOTE: this test is currently moot, as the default behavior is to keep the description if its not a placeholder +# but I leave the test in case we want to reintroduce a fine grained osmosis_keep_description meta attr def test_update_undocumented_columns_with_prior_knowledge_add_progenitor_to_meta_and_osmosis_keep_description( yaml_context: YamlRefactorContext, ): @@ -488,8 +490,7 @@ def test_update_undocumented_columns_with_prior_knowledge_add_progenitor_to_meta } # unify tags # upstream => my_tag1, my_tag2 - # local => might be empty => or we can set them if we want - # let's assume local is empty, so final is upstream + # local => is empty, so final is upstream assert set(cid.tags) == {"my_tag1", "my_tag2"} # 5) Assert YAML