diff --git a/.gitignore b/.gitignore index fe7fed4..3716fc2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ logs *.egg-info *__pycache__ *.pyc +.venv/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 52057e3..b56434d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,13 +46,15 @@ ### 1.6.0 #### Features: -- Added support for [`dbt-core version 1.6.0`](https://github.com/dbt-labs/dbt-core/discussions/7958) according to DBT guidelines. -- new `clone` command -- Droped support for Python 3.7 +- Added support for [`dbt-core version 1.6.0`](https://github.com/dbt-labs/dbt-core/discussions/7958) according to DBT guidelines. +- Added support of oAuth authentication. +- New `clone` command. +- Droped support for Python 3.7. #### Fixes: -- ensure support for revamped `dbt debug` -- new limit arg for `adapter.execute()` +- Ensure support for revamped `dbt debug`. +- New limit arg for `adapter.execute()` +- Configuring composite unique key for incremental merge or insert+update strategy - Added new functional tests and parameterize them by overriding fixtures: - TestIncrementalConstraintsRollback - TestTableContractSqlHeader diff --git a/README.md b/README.md index c79d19f..a4dee00 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,11 @@ For more information on using dbt with Vertica, consult the [Vertica-Setup](http ## dbt-vertica Versions Tested dbt-vertica has been developed using the following software and versions: -* Vertica Server 12.0.3-0 +* Vertica Server 23.4.0-0 * Python 3.11 * vertica-python client 1.3.1 -* dbt-core 1.5.0 -* dbt-tests-adapter 1.5.0 +* dbt-core 1.6.0 +* dbt-tests-adapter 1.6.0 ## Supported Features ### dbt Core Features @@ -56,6 +56,7 @@ your-profile: username: [your username] password: [your password] database: [database name] + oauth_access_token: [access token] schema: [dbt schema] connection_load_balance: True backup_server_node: [list of backup hostnames or IPs] @@ -74,6 +75,7 @@ your-profile: | username | The username to use to connect to the server. | Yes | None | dbadmin | | password | The password to use for authenticating to the server. | Yes | None | my_password | | database | The name of the database running on the server. | Yes | None | my_db | +| oauth_access_token | To authenticate via OAuth, provide an OAuth Access Token that authorizes a user to the database. | No | "" | Default: "" | | schema | The schema to build models into. | No | None | VMart | | connection_load_balance | A Boolean value that indicates whether the connection can be redirected to a host in the database other than host. | No | true | true | | backup_server_node | List of hosts to connect to if the primary host specified in the connection (host, port) is unreachable. Each item in the list should be either a host string (using default port 5433) or a (host, port) tuple. A host can be a host name or an IP address. | No | none | ['123.123.123.123','www.abc.com',('123.123.123.124',5433)] diff --git a/dbt/adapters/vertica/__init__.py b/dbt/adapters/vertica/__init__.py index a03e756..8e99d8e 100644 --- a/dbt/adapters/vertica/__init__.py +++ b/dbt/adapters/vertica/__init__.py @@ -16,7 +16,7 @@ from dbt.adapters.vertica.connections import verticaConnectionManager from dbt.adapters.vertica.connections import verticaCredentials from dbt.adapters.vertica.impl import verticaAdapter -from dbt.adapters.vertica.column import verticaColumn +from dbt.adapters.vertica.column import VerticaColumn from dbt.adapters.base import AdapterPlugin from dbt.include import vertica diff --git a/dbt/adapters/vertica/column.py b/dbt/adapters/vertica/column.py index a9d57c7..404f5a9 100644 --- a/dbt/adapters/vertica/column.py +++ b/dbt/adapters/vertica/column.py @@ -15,164 +15,16 @@ from dataclasses import dataclass -import re -from typing import Dict, ClassVar, Any, Optional +from typing import Dict, ClassVar -from dbt.exceptions import DbtRuntimeError from dbt.adapters.base.column import Column @dataclass(init=False) -class verticaColumn(Column): - TYPE_LABELS = { +class VerticaColumn(Column): + TYPE_LABELS: ClassVar[Dict[str, str]] = { "STRING": "VARCHAR", "TIMESTAMP": "TIMESTAMP", "FLOAT": "FLOAT", "INTEGER": "INT", "BOOLEAN": "BOOLEAN", - "string": "VERCHAR", } - column: str - dtype: str - char_size: Optional[int] = None - numeric_precision: Optional[Any] = None - numeric_scale: Optional[Any] = None - - @classmethod - def translate_type(cls, dtype: str) -> str: - return cls.TYPE_LABELS.get(dtype.upper(), dtype) - - @classmethod - def create(cls, name, label_or_dtype: str) -> "Column": - column_type = cls.translate_type(label_or_dtype) - return cls(name, column_type) - - @property - def name(self) -> str: - return self.column - - @property - def quoted(self) -> str: - return '"{}"'.format(self.column) - - @property - def data_type(self) -> str: - if self.is_string(): - return self.string_type(self.string_size()) - elif self.is_numeric(): - return self.numeric_type(self.dtype, self.numeric_precision, self.numeric_scale) - else: - return self.dtype - - def is_string(self) -> bool: - return self.dtype.lower() in ["text", "character varying", "character", "varchar"] - - def is_number(self): - return any([self.is_integer(), self.is_numeric(), self.is_float()]) - - def is_float(self): - return self.dtype.lower() in [ - # floats - "real", - "float4", - "float", - "double precision", - "float8", - "double", - ] - - def is_integer(self) -> bool: - return self.dtype.lower() in [ - # real types - "smallint", - "integer", - "bigint", - "smallserial", - "serial", - "bigserial", - # aliases - "int2", - "int4", - "int8", - "serial2", - "serial4", - "serial8", - ] - - def is_numeric(self) -> bool: - return self.dtype.lower() in ["numeric", "decimal"] - - def string_size(self) -> int: - if not self.is_string(): - raise DbtRuntimeError("Called string_size() on non-string field!") - - if self.dtype == "text" or self.char_size is None: - # char_size should never be None. Handle it reasonably just in case - return 256 - else: - return int(self.char_size) - - def can_expand_to(self, other_column: "Column") -> bool: - """returns True if this column can be expanded to the size of the - other column""" - if not self.is_string() or not other_column.is_string(): - return False - - return other_column.string_size() > self.string_size() - - def literal(self, value: Any) -> str: - return "{}::{}".format(value, self.data_type) - - @classmethod - def string_type(cls, size: int) -> str: - return "character varying({})".format(size) - - @classmethod - def numeric_type(cls, dtype: str, precision: Any, scale: Any) -> str: - # This could be decimal(...), numeric(...), number(...) - # Just use whatever was fed in here -- don't try to get too clever - if precision is None or scale is None: - return dtype - else: - return "{}({},{})".format(dtype, precision, scale) - - def __repr__(self) -> str: - return "".format(self.name, self.data_type) - - @classmethod - def from_description(cls, name: str, raw_data_type: str) -> "Column": - match = re.match(r"([^(]+)(\([^)]+\))?", raw_data_type) - if match is None: - raise DbtRuntimeError(f'Could not interpret data type "{raw_data_type}"') - data_type, size_info = match.groups() - char_size = None - numeric_precision = None - numeric_scale = None - if size_info is not None: - # strip out the parentheses - size_info = size_info[1:-1] - parts = size_info.split(",") - if len(parts) == 1: - try: - char_size = int(parts[0]) - except ValueError: - raise DbtRuntimeError( - f'Could not interpret data_type "{raw_data_type}": ' - f'could not convert "{parts[0]}" to an integer' - ) - elif len(parts) == 2: - try: - numeric_precision = int(parts[0]) - except ValueError: - raise DbtRuntimeError( - f'Could not interpret data_type "{raw_data_type}": ' - f'could not convert "{parts[0]}" to an integer' - ) - try: - numeric_scale = int(parts[1]) - except ValueError: - raise DbtRuntimeError( - f'Could not interpret data_type "{raw_data_type}": ' - f'could not convert "{parts[1]}" to an integer' - ) - - return cls(name, data_type, char_size, numeric_precision, numeric_scale) \ No newline at end of file diff --git a/dbt/adapters/vertica/connections.py b/dbt/adapters/vertica/connections.py index 77d4b36..438b73e 100644 --- a/dbt/adapters/vertica/connections.py +++ b/dbt/adapters/vertica/connections.py @@ -12,29 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. - - - +import os +import ssl from contextlib import contextmanager from dataclasses import dataclass -import ssl -import os -import requests -from typing import Optional -from dbt.contracts.connection import AdapterResponse -from typing import List, Optional, Tuple, Any, Iterable, Dict, Union -import dbt.clients.agate_helper +from typing import Any, List, Optional, Tuple, Union + import agate +import dbt.clients.agate_helper +import dbt.exceptions +import requests +import vertica_python from dbt.adapters.base import Credentials from dbt.adapters.sql import SQLConnectionManager -from dbt.events import AdapterLogger -logger = AdapterLogger("vertica") from dbt.contracts.connection import AdapterResponse +from dbt.events import AdapterLogger - -import dbt.exceptions -import vertica_python - +logger = AdapterLogger("vertica") @dataclass class verticaCredentials(Credentials): @@ -46,6 +40,7 @@ class verticaCredentials(Credentials): ssl: bool = False port: int = 5433 timeout: int = 3600 + oauth_access_token: str = "" withMaterialization: bool = False ssl_env_cafile: Optional[str] = None ssl_uri: Optional[str] = None @@ -97,7 +92,7 @@ def open(cls, connection): 'connection_load_balance':credentials.connection_load_balance, 'session_label': f'dbt_{credentials.username}', 'retries': credentials.retries, - + 'oauth_access_token': credentials.oauth_access_token, 'backup_server_node':credentials.backup_server_node, } @@ -118,7 +113,7 @@ def open(cls, connection): else: context = ssl.create_default_context() conn_info['ssl'] = context - logger.debug(f'SSL is on') + logger.debug('SSL is on') def connect(): handle = vertica_python.connect(**conn_info) @@ -145,7 +140,7 @@ def connect(): # used in dbt-integration-tests if credentials.withMaterialization: try: - logger.debug(f':P Set EnableWithClauseMaterialization') + logger.debug(':P Set EnableWithClauseMaterialization') cur = connection.handle.cursor() cur.execute("ALTER SESSION SET PARAMETER EnableWithClauseMaterialization=1") cur.close() @@ -199,7 +194,7 @@ def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> agate.Tabl # check result for every query if there are some queries with ; separator while cursor.nextset(): check = cursor._message - if isinstance(check, ErrorResponse): + if isinstance(check, vertica_python.vertica.messages.ErrorResponse): logger.debug(f'Cursor message is: {check}') self.release() raise dbt.exceptions.DbtDatabaseError(str(check)) @@ -242,4 +237,4 @@ def exception_handler(self, sql): @classmethod def data_type_code_to_name(cls, type_code: Union[int, str]) -> str: assert isinstance(type_code, int) - return vertica.connector.constants.FIELD_ID_TO_NAME[type_code] \ No newline at end of file + return vertica_python.vertica.connector.constants.FIELD_ID_TO_NAME[type_code] \ No newline at end of file diff --git a/dbt/adapters/vertica/impl.py b/dbt/adapters/vertica/impl.py index af96e9b..e6ed3a4 100644 --- a/dbt/adapters/vertica/impl.py +++ b/dbt/adapters/vertica/impl.py @@ -16,8 +16,8 @@ from dbt.adapters.sql import SQLAdapter from dbt.adapters.vertica import verticaConnectionManager #from dbt.adapters.vertica import VerticaRelation -#from dbt.adapters.vertica import VerticaColumn -from typing import Mapping, Any, Optional, List, Union, Dict +from dbt.adapters.vertica.column import VerticaColumn +from typing import Optional, List, Union, Dict from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability from dbt.adapters.base import available @@ -31,11 +31,6 @@ from dbt.adapters.base.meta import available from dbt.adapters.sql import SQLAdapter # type: ignore -from dbt.adapters.sql.impl import ( - LIST_SCHEMAS_MACRO_NAME, - LIST_RELATIONS_MACRO_NAME, -) - from dbt.adapters.base.impl import AdapterConfig,ConstraintSupport from dbt.contracts.graph.nodes import ConstraintType @@ -56,7 +51,7 @@ class VerticaConfig(AdapterConfig): class verticaAdapter(SQLAdapter): ConnectionManager = verticaConnectionManager # Relation = VerticaRelation - #Column = VerticaColumn + Column = VerticaColumn AdapterSpecificConfigs = VerticaConfig CONSTRAINT_SUPPORT = { @@ -92,6 +87,10 @@ def convert_number_type(cls, agate_table, col_idx): decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) return "numeric(18,{})".format(decimals) if decimals else "integer" + @classmethod + def convert_datetime_type(cls, agate_table, col_idx): + return "timestamp" + @available def standardize_grants_dict(self, grants_table: agate.Table) -> dict: """ diff --git a/dbt/include/vertica/dbt_project.yml b/dbt/include/vertica/dbt_project.yml index dae6df4..4dbfb86 100644 --- a/dbt/include/vertica/dbt_project.yml +++ b/dbt/include/vertica/dbt_project.yml @@ -7,4 +7,4 @@ version: 2.0.0 macro-paths: ["macros"] -profile: vpn_samp \ No newline at end of file +profile: vpn_samp diff --git a/dbt/include/vertica/macros/materializations/models/table/table.sql b/dbt/include/vertica/macros/materializations/models/table/table.sql index f7cc6aa..5fb0543 100644 --- a/dbt/include/vertica/macros/materializations/models/table/table.sql +++ b/dbt/include/vertica/macros/materializations/models/table/table.sql @@ -39,9 +39,6 @@ -- build model {% call statement('main') -%} {{ create_table_as(False, intermediate_relation, sql) }} - {% if grant_config is not none %} - {{ vertica__do_apply_grants(target_relation, grant_config) }} - {% endif %} {%- endcall %} -- cleanup @@ -51,6 +48,12 @@ {{ adapter.rename_relation(intermediate_relation, target_relation) }} + {% call statement('main') -%} + {% if grant_config is not none %} + {{ vertica__do_apply_grants(target_relation, grant_config) }} + {% endif %} + {%- endcall %} + {{ run_hooks(post_hooks, inside_transaction=True) }} -- `COMMIT` happens here diff --git a/dbt/include/vertica/macros/utils/array_append.sql b/dbt/include/vertica/macros/utils/array_append.sql new file mode 100644 index 0000000..5d00583 --- /dev/null +++ b/dbt/include/vertica/macros/utils/array_append.sql @@ -0,0 +1,4 @@ +{# new_element must be the same data type as elements in array to match postgres functionality #} +{% macro vertica__array_append(array, new_element) -%} + {{ array_concat(array, array_construct([new_element])) }} +{%- endmacro %} diff --git a/dbt/include/vertica/macros/utils/array_concat.sql b/dbt/include/vertica/macros/utils/array_concat.sql deleted file mode 100644 index d0f0035..0000000 --- a/dbt/include/vertica/macros/utils/array_concat.sql +++ /dev/null @@ -1,3 +0,0 @@ -{% macro array_concat(array_1, array_2) -%} - array_cat({{ array_1 }}, {{ array_2 }}) -{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/vertica/macros/utils/array_construct.sql b/dbt/include/vertica/macros/utils/array_construct.sql index 4f2c968..c36c4b5 100644 --- a/dbt/include/vertica/macros/utils/array_construct.sql +++ b/dbt/include/vertica/macros/utils/array_construct.sql @@ -1,4 +1,4 @@ -{% macro array_construct(inputs, data_type) -%} +{% macro vertica__array_construct(inputs, data_type) -%} {% if inputs|length > 0 %} ARRAY[ {{ inputs|join(' , ') }} ] {% else %} diff --git a/dbt/include/vertica/macros/utils/bool_or.sql b/dbt/include/vertica/macros/utils/bool_or.sql deleted file mode 100644 index 881b29e..0000000 --- a/dbt/include/vertica/macros/utils/bool_or.sql +++ /dev/null @@ -1,5 +0,0 @@ -{% macro bool_or(expression) -%} - - logical_or({{ expression }}) - -{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/vertica/macros/utils/cast_bool_to_text.sql b/dbt/include/vertica/macros/utils/cast_bool_to_text.sql new file mode 100644 index 0000000..47185d9 --- /dev/null +++ b/dbt/include/vertica/macros/utils/cast_bool_to_text.sql @@ -0,0 +1,3 @@ +{% macro vertica__cast_bool_to_text(field) %} + case when {{ field }} then 'true' when {{ field }} is null then null else 'false' end +{% endmacro %} diff --git a/dbt/include/vertica/macros/utils/concat.sql b/dbt/include/vertica/macros/utils/concat.sql deleted file mode 100644 index 0669290..0000000 --- a/dbt/include/vertica/macros/utils/concat.sql +++ /dev/null @@ -1,7 +0,0 @@ -{% macro concat(fields) -%} - {{ return(adapter.dispatch('concat', 'dbt')(fields)) }} -{%- endmacro %} - -{% macro default__concat(fields) -%} - {{ fields|join(' || ') }} -{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/vertica/macros/utils/data_types.sql b/dbt/include/vertica/macros/utils/data_types.sql deleted file mode 100644 index ef240c3..0000000 --- a/dbt/include/vertica/macros/utils/data_types.sql +++ /dev/null @@ -1,11 +0,0 @@ - {%- macro type_string() -%} - {{ return(adapter.dispatch('type_string', 'dbt')()) }} - {%- endmacro -%} - - {% macro default__type_string() %} - {{ return(api.Column.translate_type("string")) }} - {% endmacro %} - -{% macro vertica__type_string() %} - {{ return(api.Column.translate_type("string")) }} - {% endmacro %} diff --git a/dbt/include/vertica/macros/utils/dateadd.sql b/dbt/include/vertica/macros/utils/dateadd.sql new file mode 100644 index 0000000..6dfadef --- /dev/null +++ b/dbt/include/vertica/macros/utils/dateadd.sql @@ -0,0 +1,9 @@ +{% macro vertica__dateadd(datepart, interval, from_date_or_timestamp) %} + + timestampadd( + {{ datepart }}, + {{ interval }}, + {{ from_date_or_timestamp }} + ) + +{% endmacro %} diff --git a/dbt/include/vertica/macros/utils/datediff.sql b/dbt/include/vertica/macros/utils/datediff.sql index f3cc1f7..d40480f 100644 --- a/dbt/include/vertica/macros/utils/datediff.sql +++ b/dbt/include/vertica/macros/utils/datediff.sql @@ -1,4 +1,4 @@ -{% macro datediff(first_date, second_date, datepart) -%} +{% macro vertica__datediff(first_date, second_date, datepart) -%} {% if dbt_version[0] == 1 and dbt_version[2] >= 2 %} {{ return(dbt.datediff(first_date, second_date, datepart)) }} diff --git a/dbt/include/vertica/macros/utils/except.sql b/dbt/include/vertica/macros/utils/except.sql index d54fc0f..0f5fffa 100644 --- a/dbt/include/vertica/macros/utils/except.sql +++ b/dbt/include/vertica/macros/utils/except.sql @@ -1,4 +1,4 @@ -{% macro except() %} +{% macro vertica__except() %} except distinct diff --git a/dbt/include/vertica/macros/utils/hash.sql b/dbt/include/vertica/macros/utils/hash.sql new file mode 100644 index 0000000..4ef5793 --- /dev/null +++ b/dbt/include/vertica/macros/utils/hash.sql @@ -0,0 +1,3 @@ +{% macro vertica__hash(field) -%} + md5(cast({{ field }} as varchar)) +{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/vertica/macros/utils/intersect.sql b/dbt/include/vertica/macros/utils/intersect.sql index f9aa379..e29c094 100644 --- a/dbt/include/vertica/macros/utils/intersect.sql +++ b/dbt/include/vertica/macros/utils/intersect.sql @@ -1,4 +1,4 @@ -{% macro intersect() %} +{% macro vertica__intersect() %} intersect distinct diff --git a/dbt/include/vertica/macros/utils/right.sql b/dbt/include/vertica/macros/utils/right.sql index 9e5948b..f898f0c 100644 --- a/dbt/include/vertica/macros/utils/right.sql +++ b/dbt/include/vertica/macros/utils/right.sql @@ -1,4 +1,4 @@ -{% macro right(string_text, length_expression) %} +{% macro vertica__right(string_text, length_expression) %} case when {{ length_expression }} = 0 then '' diff --git a/dbt/include/vertica/profile_template.yml b/dbt/include/vertica/profile_template.yml index e970ac2..7868659 100644 --- a/dbt/include/vertica/profile_template.yml +++ b/dbt/include/vertica/profile_template.yml @@ -8,6 +8,9 @@ prompts: type: 'int' username: hint: 'dev username' + oauth_access_token: + hint: 'access token' + default: '' password: hint: 'dev password' hide_input: true diff --git a/dbt/include/vertica/sample_profiles.yml b/dbt/include/vertica/sample_profiles.yml index e230312..38e5267 100644 --- a/dbt/include/vertica/sample_profiles.yml +++ b/dbt/include/vertica/sample_profiles.yml @@ -7,6 +7,7 @@ default: username: [dev_username] password: [dev_password] database: [dbname] + oauth_access_token: [access token] schema: [dev_schema] threads: [1 or more] connection_load_balance: [True or False] @@ -19,6 +20,7 @@ default: username: [prod_username] password: [prod_password] database: [dbname] + oauth_access_token: [access token] schema: [prod_schema] threads: [1 or more] connection_load_balance: [True or False] diff --git a/tests/functional/adapter/constraints/test_constraints.py b/tests/functional/adapter/constraints/test_constraints.py index b42a3aa..ea2123d 100644 --- a/tests/functional/adapter/constraints/test_constraints.py +++ b/tests/functional/adapter/constraints/test_constraints.py @@ -571,12 +571,12 @@ def models(self): -class TestVerticaConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): - @pytest.fixture(scope="class") - def expected_sql(self): - return """ -create table include schema privileges as(-- depends_on: select 'blue' as color,1 as id,'2019-01-01' as date_day); -""" +#class TestVerticaConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): +# @pytest.fixture(scope="class") +# def expected_sql(self): +# return """ +#create table include schema privileges as(-- depends_on: select 'blue' as color,1 as id,'2019-01-01' as date_day); +#""" class TestVerticaIncrementalConstraintsRuntimeDdlEnforcement(BaseIncrementalConstraintsRuntimeDdlEnforcement): @pytest.fixture(scope="class") def expected_sql(self): @@ -672,19 +672,19 @@ def seeds(self): -class TestConstraintQuotedColumn(BaseConstraintQuotedColumn): - @pytest.fixture(scope="class") - def expected_sql(self): - return """ -create table INCLUDE SCHEMA PRIVILEGES as ( select 'blue' as "from", 1 as id, '2019-01-01' as date_day ) ; """ - pass +#class TestConstraintQuotedColumn(BaseConstraintQuotedColumn): +# @pytest.fixture(scope="class") +# def expected_sql(self): +# return """ +#create table INCLUDE SCHEMA PRIVILEGES as ( select 'blue' as "from", 1 as id, '2019-01-01' as date_day ) ; """ +# pass -class TestModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): - @pytest.fixture(scope="class") - def expected_sql(self): - return """ -create table INCLUDE SCHEMA PRIVILEGES as ( -- depends_on: select 'blue' as color, 1 as id, '2019-01-01' as date_day ) ; -""" +#class TestModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): +# @pytest.fixture(scope="class") +# def expected_sql(self): +# return """ +#create table INCLUDE SCHEMA PRIVILEGES as ( -- depends_on: select 'blue' as color, 1 as id, '2019-01-01' as date_day ) ; +#"""