S3x version3 (#126)

Co-authored-by: sachinthakur96 <[email protected]>

.github/workflows/vertica-test.yml

@@ -58,6 +58,27 @@ jobs:
       run: python -m pytest tests/functional/adapter/utils/test_concat.py
     - name: Test Data Type Int
       run: python -m pytest tests/functional/adapter/utils/data_type/
+    - name: Test Last Relation Modified
+      run: python -m pytest tests/functional/adapter/test_get_last_relation_modified.py
+    - name: Test  Relation Caching
+      run: python -m pytest tests/functional/adapter/test_list_relations_without_caching.py
+    - name: Test Store test Failure
+      run: python -m pytest tests/functional/adapter/store_test_failures_tests/test_store_test_failures.py
+    - name: Test Date Spine
+      run: python -m pytest tests/functional/adapter/utils/test_date_spine.py
+    - name: Test DBT Show
+      run: python -m pytest tests/functional/adapter/dbt-show/test_dbt_show.py
+    - name: Test DBT Clone
+      run: python -m pytest tests/functional/adapter/dbt_clone/
+    - name: Test Seeds
+      run: python -m pytest tests/functional/adapter/seeds/test_seeds.py
+
+
+
+
+
+
+
 
 
 

CHANGELOG.md

@@ -2,6 +2,47 @@
 - This file provides a full account of all changes to dbt-vertica.
 - "Breaking changes" listed under a version may require action from end users.
 
+
+
+### 1.7.0
+
+#### Features:
+- New capability support structure for adapters
+- Metadata freshness checks
+- Catalog fetch performance improvements
+- Behavior of dbt show's --limit flag
+- Migrate date_spine() Macro from dbt-utils to Core
+- Data Spine Tests
+- Storing Test Failures as View
+- Additional Tests
+
+#### Fixes:
+
+- Metadata freshness checks Tests
+  - TestGetLastRelationModified
+  - TestListRelationsWithoutCachingSingle
+  - TestListRelationsWithoutCachingFull
+- Behavior of dbt show's --limit flag Tests
+  - BaseShowSqlHeader
+  - BaseShowLimit
+- Data Spine Tests
+  - TestDateSpine
+  - TestGenerateSeries
+  - TestGetIntervalsBetween
+  - TestGetPowersOfTwo
+- Storing Test Failures as View
+  - TestStoreTestFailuresAsInteractions
+  - TestStoreTestFailuresAsProjectLevelOff
+  - TestStoreTestFailuresAsProjectLevelView
+  - TestStoreTestFailuresAsGeneric
+  - TestStoreTestFailuresAsProjectLevelEphemeral
+  - TestStoreTestFailuresAsExceptions
+- Additional Tests
+  - TestCloneSameTargetAndState
+  - SeedUniqueDelimiterTestBase 
+  - TestSeedWithWrongDelimiter
+  - TestSeedWithEmptyDelimiter
+
 ### 1.6.0
 
 #### Features:

dbt/adapters/vertica/__version__.py

@@ -14,4 +14,4 @@
 
 
 
-version = "1.6.0"
+version = "1.7.3"

dbt/include/vertica/macros/adapters/catalog.sql

@@ -0,0 +1,162 @@
+{% macro vertica__get_catalog(information_schema, schemas) -%}
+
+
+   {% set query %}     
+
+        with tables as (
+              {{ vertica__get_catalog_tables_sql(information_schema) }}
+            {{ vertica__get_catalog_schemas_where_clause_sql(schemas) }}
+             ), 
+        columns as  (
+             {{ vertica__get_catalog_columns_sql(information_schema) }}
+             {{ vertica__get_catalog_schemas_where_clause_sql(schemas) }}
+
+        )
+
+        {{vertica__get_catalog_results_sql ()}}
+
+         {%- endset -%}
+          {{ return(run_query(query)) }}
+
+
+
+{%- endmacro %}
+
+
+
+
+
+
+{% macro vertica__get_catalog_relations(information_schema, relations) -%}
+   {% set query %}
+
+
+            with tables as (
+                {{ vertica__get_catalog_tables_sql(information_schema) }}
+                {{ vertica__get_catalog_relations_where_clause_sql(relations) }}
+             ), 
+        columns as (
+             {{ vertica__get_catalog_columns_sql(information_schema) }}
+             {{ vertica__get_catalog_relations_where_clause_sql(relations) }}
+
+        )
+    {{vertica__get_catalog_results_sql ()}}
+         {%- endset -%}
+
+          {{ return(run_query(query)) }}
+
+{%- endmacro %}
+
+
+
+
+
+
+
+
+{% macro vertica__get_catalog_tables_sql(information_schema) -%}
+   select * from  (
+ select
+    '{{ information_schema.database }}' table_database
+    , tab.table_schema
+    , tab.table_name
+    , 'TABLE' table_type
+    , comment table_comment
+    , tab.owner_name table_owner
+    , col.column_name
+    , col.ordinal_position column_index
+    , col.data_type column_type
+    , nullif('','') column_comment
+    from v_catalog.tables tab
+    join v_catalog.columns col on tab.table_id = col.table_id
+    left join v_catalog.comments on tab.table_id = object_id
+    union all
+    select
+    '{{ information_schema.database }}' table_database
+    , vw.table_schema
+    , vw.table_name
+    , 'VIEW' table_type
+    , comment table_comment
+    , vw.owner_name table_owner
+    , col.column_name
+    , col.ordinal_position column_index
+    , col.data_type column_type
+    , nullif('','') column_comment
+    from v_catalog.views vw
+    join v_catalog.view_columns col on vw.table_id = col.table_id
+    left join v_catalog.comments on vw.table_id = object_id
+
+   )  anything
+{%- endmacro %}
+
+
+
+{% macro vertica__get_catalog_columns_sql(information_schema) -%}
+    select * from  ( select
+       '{{ information_schema.database }}' table_database , 
+    table_schema 
+    ,table_name
+    , column_name
+    , ordinal_position column_index
+    , data_type column_type
+
+    from v_catalog.columns 
+
+         union all
+
+     select 
+    '{{ information_schema.database }}' table_database ,
+    table_schema 
+    ,table_name
+    ,  column_name
+    , ordinal_position column_index
+    , data_type column_type
+
+    from v_catalog.view_columns
+    )  y
+{%- endmacro %}
+
+{% macro vertica__get_catalog_results_sql() -%}
+    select *
+    from tables
+    join columns using ("table_database", "table_schema", "table_name");
+
+{%- endmacro %}
+
+{% macro vertica__get_catalog_schemas_where_clause_sql(schemas) -%}
+        where   
+        (
+          {%- for schema in schemas -%} 
+
+
+            lower(table_schema) = lower('{{ schema }}') {%- if not loop.last %} or {% endif %}
+          {%- endfor -%}
+        )
+
+         order by table_schema, table_name, column_index
+
+{%- endmacro %}
+
+
+{% macro vertica__get_catalog_relations_where_clause_sql(relations) -%}
+    where (
+        {%- for relation in relations -%}
+            {% if relation.schema and relation.identifier %}
+                (
+                    upper("table_schema") = upper('{{ relation.schema }}')
+                    and upper("table_name") = upper('{{ relation.identifier }}')
+                )
+            {% elif relation.schema %}
+                (
+                    upper("table_schema") = upper('{{ relation.schema }}')
+                )
+            {% else %}
+                {% do exceptions.raise_compiler_error(
+                    '`get_catalog_relations` requires a list of relations, each with a schema'
+                ) %}
+            {% endif %}
+
+            {%- if not loop.last %} or {% endif -%}
+        {%- endfor -%}
+    )
+{%- endmacro %}

dbt/include/vertica/macros/adapters/metadata.sql

@@ -1,52 +1,4 @@
-{% macro vertica__get_catalog(information_schema, schemas) -%}
-  {% call statement('get_catalog', fetch_result=True) %}
 
-    select
-    '{{ information_schema.database }}' table_database
-    , tab.table_schema
-    , tab.table_name
-    , 'TABLE' table_type
-    , comment table_comment
-    , tab.owner_name table_owner
-    , col.column_name
-    , col.ordinal_position column_index
-    , col.data_type column_type
-    , nullif('','') column_comment
-    from v_catalog.tables tab
-    join v_catalog.columns col on tab.table_id = col.table_id
-    left join v_catalog.comments on tab.table_id = object_id
-    where not(tab.is_system_table) and
-        (
-          {%- for schema in schemas -%}
-            lower(tab.table_schema) = lower('{{ schema }}') {%- if not loop.last %} or {% endif %}
-          {%- endfor -%}
-        )
-    union all
-    select
-    '{{ information_schema.database }}' table_database
-    , vw.table_schema
-    , vw.table_name
-    , 'VIEW' table_type
-    , comment table_comment
-    , vw.owner_name table_owner
-    , col.column_name
-    , col.ordinal_position column_index
-    , col.data_type column_type
-    , nullif('','') column_comment
-    from v_catalog.views vw
-    join v_catalog.view_columns col on vw.table_id = col.table_id
-    left join v_catalog.comments on vw.table_id = object_id
-    where not(vw.is_system_view) and
-        (
-          {%- for schema in schemas -%}
-            lower(vw.table_schema) = lower('{{ schema }}') {%- if not loop.last %} or {% endif %}
-          {%- endfor -%}
-        )
-    order by table_schema, table_name, column_index
-
-  {% endcall %}
-  {{ return(load_result('get_catalog').table) }}
-{% endmacro %}
 
 
 {% macro vertica__information_schema_name(database) -%}
@@ -96,4 +48,30 @@
     where table_schema ilike '{{ schema_relation.schema }}'
   {% endcall %}
   {{ return(load_result('list_relations_without_caching').table) }}
-{% endmacro %}
+{% endmacro %}
+
+
+
+{% macro vertica__get_relation_last_modified(information_schema, relations) -%}
+
+  {%- call statement('last_modified', fetch_result=True) -%}
+        select table_schema as schema,
+              table_name as identifier,
+              create_time as last_modified,
+               {{ current_timestamp() }} as snapshotted_at 
+        from v_catalog.tables
+        where (
+          {%- for relation in relations -%}
+            (upper(table_schema) = upper('{{ relation.schema }}') and
+             upper(table_name) = upper('{{ relation.identifier }}')){%- if not loop.last %} or {% endif -%}
+          {%- endfor -%}
+        )
+  {%- endcall -%}
+
+  {{ return(load_result('last_modified')) }}
+
+{% endmacro %}
+
+
+
+

dbt/include/vertica/macros/utils/array_construct.sql

@@ -1,7 +1,7 @@
 {% macro vertica__array_construct(inputs, data_type) -%}
     {% if inputs|length > 0 %}
-    [ {{ inputs|join(' , ') }} ]
+    ARRAY[ {{ inputs|join(' , ') }} ]
     {% else %}
-    ARRAY<{{data_type}}>[]
+    ARRAY[{{data_type}}]
     {% endif %}
 {%- endmacro %}