diff --git a/src/dbt_osmosis/core/osmosis.py b/src/dbt_osmosis/core/osmosis.py index 46b783c..fed19bc 100644 --- a/src/dbt_osmosis/core/osmosis.py +++ b/src/dbt_osmosis/core/osmosis.py @@ -306,19 +306,35 @@ class YamlRefactorSettings: """Settings for yaml based refactoring operations.""" fqn: str | None = None + """Filter models to action via a fully qualified name match.""" models: list[str] = field(default_factory=list) + """Filter models to action via a file path match.""" dry_run: bool = False + """Do not write changes to disk.""" catalog_file: str | None = None + """Path to the dbt catalog.json file to use preferentially instead of live warehouse introspection""" skip_add_columns: bool = False + """Skip adding missing columns in the yaml files.""" skip_add_tags: bool = False + """Skip appending upstream tags in the yaml files.""" skip_add_data_types: bool = False + """Skip adding data types in the yaml files.""" numeric_precision: bool = False + """Include numeric precision in the data type.""" char_length: bool = False + """Include character length in the data type.""" skip_merge_meta: bool = False + """Skip merging upstream meta fields in the yaml files.""" add_progenitor_to_meta: bool = False + """Add a custom progenitor field to the meta section indicating a column's origin.""" use_unrendered_descriptions: bool = False + """Use unrendered descriptions preserving things like {{ doc(...) }} which are otherwise pre-rendered in the manifest object""" add_inheritance_for_specified_keys: list[str] = field(default_factory=list) + """Include additional keys in the inheritance process.""" output_to_lower: bool = False + """Force column name and data type output to lowercase in the yaml files.""" + force_inherit_descriptions: bool = False + """Force inheritance of descriptions from upstream models, even if node has a valid description.""" @dataclass @@ -1117,7 +1133,9 @@ def _build_column_knowledge_graph( if incoming_val := graph_edge.pop(inheritable, current_val): graph_edge[inheritable] = incoming_val - if graph_edge.get("description", EMPTY_STRING) in context.placeholders: + if graph_edge.get("description", EMPTY_STRING) in context.placeholders or ( + generation == "generation_0" and context.settings.force_inherit_descriptions + ): _ = graph_edge.pop("description", None) if graph_edge.get("tags") == []: del graph_edge["tags"] @@ -1210,8 +1228,6 @@ def remove_columns_not_in_database( context: YamlRefactorContext, node: ResultNode | None = None ) -> None: """Remove columns from a dbt node and it's corresponding yaml section that are not present in the database. Changes are implicitly buffered until commit_yamls is called.""" - if context.settings.skip_add_columns: - return if node is None: for _, node in filter_models(context): remove_columns_not_in_database(context, node) diff --git a/tests/test_column_level_knowledge_propagator.py b/tests/test_column_level_knowledge_propagator.py index ab1de4d..9440a00 100644 --- a/tests/test_column_level_knowledge_propagator.py +++ b/tests/test_column_level_knowledge_propagator.py @@ -13,7 +13,6 @@ DbtConfiguration, YamlRefactorContext, YamlRefactorSettings, - _build_column_knowledge_graph, _build_node_ancestor_tree, _get_member_yaml, create_dbt_project_context, @@ -158,7 +157,7 @@ def test_inherit_upstream_column_knowledge(yaml_context: YamlRefactorContext): expect[column]["granularity"] = None target_node = manifest.nodes["model.jaffle_shop_duckdb.customers"] - # NOTE: we will only update empty / placeholders descriptions by design + # NOTE: we will only update empty / placeholders descriptions by design, see force_inherit_descriptions for legacy behavior target_node.columns["customer_id"].description = "" yaml_context.placeholders = ("",) @@ -172,8 +171,7 @@ def test_inherit_upstream_column_knowledge(yaml_context: YamlRefactorContext): def test_inherit_upstream_column_knowledge_with_mutations(yaml_context: YamlRefactorContext): - yaml_context.settings.skip_add_tags = False - yaml_context.settings.skip_merge_meta = False + yaml_context.settings.force_inherit_descriptions = True # NOTE: matches legacy behavior manifest = yaml_context.project.manifest customer_id_column = manifest.nodes["model.jaffle_shop_duckdb.stg_customers"].columns[ @@ -185,9 +183,6 @@ def test_inherit_upstream_column_knowledge_with_mutations(yaml_context: YamlRefa target_node = manifest.nodes["model.jaffle_shop_duckdb.customers"] target_node_customer_id = target_node.columns["customer_id"] - target_node_customer_id.description = ( - "" # NOTE: allow inheritance to update this, otherwise a valid description would be skipped - ) target_node_customer_id.tags = ["my_tag3", "my_tag4"] target_node_customer_id.meta = {"my_key": "my_local_value", "my_new_key": "my_new_value"}