Skip to content

Commit

Permalink
feat: be more liberal in accepting config keys of dbt_osmosis such as…
Browse files Browse the repository at this point in the history
… to support config block syntax
  • Loading branch information
z3z1ma committed Dec 31, 2024
1 parent ab507b5 commit 0b62d64
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 10 deletions.
27 changes: 19 additions & 8 deletions src/dbt_osmosis/core/osmosis.py
Original file line number Diff line number Diff line change
Expand Up @@ -383,14 +383,20 @@ def mutated(self) -> bool:
@property
def source_definitions(self) -> dict[str, t.Any]:
"""The source definitions from the dbt project config."""
defs = self.project.config.vars.to_dict().get("dbt-osmosis", {}).copy()
c = self.project.config.vars.to_dict()
defs = _find_first(
[c.get(k, {}) for k in ["dbt-osmosis", "dbt_osmosis"]], lambda v: bool(v), {}
)
defs.pop(SKIP_PATTERNS, None)
return defs

@property
def skip_patterns(self) -> list[str]:
"""The column name skip patterns from the dbt project config."""
defs = self.project.config.vars.to_dict().get("dbt-osmosis", {}).copy()
c = self.project.config.vars.to_dict()
defs = _find_first(
[c.get(k, {}) for k in ["dbt-osmosis", "dbt_osmosis"]], lambda v: bool(v), {}
)
return defs.pop(SKIP_PATTERNS, [])

def read_catalog(self) -> CatalogArtifact | None:
Expand Down Expand Up @@ -716,7 +722,12 @@ def _get_yaml_path_template(context: YamlRefactorContext, node: ResultNode) -> s
if isinstance(def_or_path, dict):
return def_or_path.get("path")
return def_or_path
path_template = node.config.extra.get("dbt-osmosis", node.unrendered_config.get("dbt-osmosis"))
conf = [
c.get(k)
for k in ["dbt-osmosis", "dbt_osmosis"]
for c in [node.config.extra, node.unrendered_config]
]
path_template = _find_first(t.cast(list[str | None], conf), lambda v: v is not None)
if not path_template:
raise MissingOsmosisConfig(
f"Config key `dbt-osmosis: <path>` not set for model {node.name}"
Expand Down Expand Up @@ -1326,12 +1337,12 @@ def get_candidates(self, name: str, node: ResultNode, context: DbtProjectContext
variants = []
prefix = t.cast(
str,
node.config.extra.get(
"dbt-osmosis-prefix", node.unrendered_config.get("dbt-osmosis-prefix")
),
node.meta.get("osmosis_prefix") # Can be set in the node yml (legacy support)
or node.config.extra.get("dbt_osmosis_prefix") # Or in dbt_project.yml / {{ config() }}
or node.unrendered_config.get("dbt_osmosis_prefix"),
)
if prefix and name.startswith(prefix):
variants.append(name[len(prefix) :])
if prefix:
variants.append(name.removeprefix(prefix))
return variants


Expand Down
5 changes: 3 additions & 2 deletions tests/test_column_level_knowledge_propagator.py
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,8 @@ def test_update_undocumented_columns_with_prior_knowledge_with_osmosis_keep_desc
assert col_0["description"] == column_description_not_updated


# NOTE: this test is currently moot, as the default behavior is to keep the description if its not a placeholder
# but I leave the test in case we want to reintroduce a fine grained osmosis_keep_description meta attr
def test_update_undocumented_columns_with_prior_knowledge_add_progenitor_to_meta_and_osmosis_keep_description(
yaml_context: YamlRefactorContext,
):
Expand Down Expand Up @@ -488,8 +490,7 @@ def test_update_undocumented_columns_with_prior_knowledge_add_progenitor_to_meta
}
# unify tags
# upstream => my_tag1, my_tag2
# local => might be empty => or we can set them if we want
# let's assume local is empty, so final is upstream
# local => is empty, so final is upstream
assert set(cid.tags) == {"my_tag1", "my_tag2"}

# 5) Assert YAML
Expand Down

0 comments on commit 0b62d64

Please sign in to comment.