Merge branch 'main' into edgarrmondragon/chore/refactors-typos-cleanup

.github/ISSUE_TEMPLATE/bug.yml

@@ -0,0 +1,67 @@
+name: Bug Report
+description: File a bug report
+title: "bug: <title>"
+
+labels:
+  - bug
+
+assignees:
+  - edgarrmondragon
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: target_version
+    attributes:
+      label: Target Version
+      description: Version of the package you are using
+      placeholder: "0.0.10 on PyPI, 0.0.10 on GitHub, main branch, etc."
+    validations:
+      required: true
+  - type: dropdown
+    id: python_version
+    attributes:
+      label: Python Version
+      description: Version of Python you are using
+      options:
+        - "3.12"
+        - "3.11"
+        - "3.10"
+        - "3.9"
+        - "3.8"
+        - "NA"
+    validations:
+      required: true
+  - type: input
+    id: postgres_version
+    attributes:
+      label: PostgreSQL Version
+      description: Version of PostgreSQL the target is loading
+      placeholder: "15.1"
+    validations:
+      required: true
+  - type: input
+    id: os
+    attributes:
+      label: Operating System
+      description: What operating system you are using
+      placeholder: "Linux - Ubuntu 20.04"
+    validations:
+      required: true
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: Description
+      description: Describe what you were trying to get done
+      placeholder: Tell us what happened, what went wrong, and what you expected to happen
+    validations:
+      required: true
+  - type: input
+    id: slack_or_linen
+    attributes:
+      label: Link to Slack/Linen
+      description: Provide a link to the Slack or Linen conversation, if applicable
+      placeholder: "https://..."

.github/workflows/ci_workflow.yml

@@ -3,8 +3,20 @@ name: Test target-postgres
 on:
   pull_request:
     types: [opened, synchronize, reopened]
+    paths:
+    - docker-compose.yml
+    - pyproject.toml
+    - poetry.lock
+    - targets_postgres/**
+    - .github/workflows/ci_workflow.yml
   push:
     branches: [main]
+    paths:
+    - docker-compose.yml
+    - pyproject.toml
+    - poetry.lock
+    - targets_postgres/**
+    - .github/workflows/ci_workflow.yml
   workflow_dispatch:
     inputs: {}
 
@@ -43,6 +55,7 @@ jobs:
         pipx install poetry
     - name: Install dependencies
       run: |
+        poetry env use python${{ matrix.python-version }}
         poetry install
     - name: Run pytest
       run: |
@@ -65,14 +78,12 @@ jobs:
           --health-retries 5
         ports:
           - 5432:5432
-    env:
-      python-version: "3.8"
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
+    - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: 3.x
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/release_workflow.yml

@@ -37,4 +37,6 @@ jobs:
         tag: ${{ github.ref }}
         overwrite: true
         file_glob: true
-    - uses: pypa/[email protected]
+    - uses: pypa/[email protected]
+      with:
+        attestations: true

.pre-commit-config.yaml

@@ -18,14 +18,14 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.2
+  rev: v0.6.7
   hooks:
   - id: ruff
     args: [--fix]
   - id: ruff-format
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: 'v1.10.1'
+  rev: 'v1.11.2'
   hooks:
   - id: mypy
     exclude: tests

meltano.yml

@@ -37,17 +37,34 @@ plugins:
     settings:
     - name: sqlalchemy_url
       kind: password
+      sensitive: true
     - name: ssl_enable
       kind: boolean
+      sensitive: true
     - name: ssl_client_certificate_enable
       kind: boolean
+      sensitive: true
     - name: ssl_mode
     - name: ssl_certificate_authority
       kind: password
+      sensitive: true
     - name: ssl_client_certificate
       kind: password
+      sensitive: true
     - name: ssl_client_private_key
       kind: password
+      sensitive: true
+    - name: password
+      kind: password
+      sensitive: true
+    - name: host
+    - name: port
+      kind: integer
+    - name: user
+    - name: database
+    - name: target_schema
+    - name: add_record_metadata
+      kind: boolean
     config:
       host: localhost
       port: 5432

pyproject.toml

@@ -1,13 +1,14 @@
 [tool.poetry]
 name = "meltanolabs-target-postgres"
 version = "0.0.0"
-description = "`target-postgres` is a Singer target for Postgres, built with the Meltano SDK for Singer Targets."
+description = "Singer target for Postgres, built with the Meltano SDK for Singer Targets."
 authors = ["Meltano Team and Contributors <[email protected]>"]
 maintainers = ["Meltano Team and Contributors <[email protected]>"]
 license = "MIT"
 readme = "README.md"
 homepage = "https://meltano.com"
-repository = "https://github.com/meltanolabs/target-postgres"
+repository = "https://github.com/MeltanoLabs/target-postgres"
+documentation = "https://github.com/MeltanoLabs/target-postgres/blob/main/README.md"
 keywords = [
     "Postgres",
     "Singer",
@@ -32,12 +33,16 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.8"
+faker = {version = "~=29.0", optional = true}
 psycopg2-binary = "2.9.9"
 sqlalchemy = "~=2.0"
 sshtunnel = "0.4.0"
 
 [tool.poetry.dependencies.singer-sdk]
-version = "~=0.38.0"
+version = "~=0.40.0a1"
+
+[tool.poetry.extras]
+faker = ["faker"]
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.4.2"
@@ -58,7 +63,7 @@ module = ["sshtunnel"]
 ignore_missing_imports = true
 
 [build-system]
-requires = ["poetry-core==1.9.0", "poetry-dynamic-versioning==1.3.0"]
+requires = ["poetry-core==1.9.0", "poetry-dynamic-versioning==1.4.0"]
 build-backend = "poetry_dynamic_versioning.backend"
 
 [tool.poetry.scripts]

target_postgres/connector.py

@@ -35,6 +35,9 @@
 )
 from sshtunnel import SSHTunnelForwarder
 
+if t.TYPE_CHECKING:
+    from singer_sdk.connectors.sql import FullyQualifiedName
+
 
 class PostgresConnector(SQLConnector):
     """Sets up SQL Alchemy, and other Postgres related stuff."""
@@ -96,7 +99,7 @@ def interpret_content_encoding(self) -> bool:
 
     def prepare_table(  # type: ignore[override]  # noqa: PLR0913
         self,
-        full_table_name: str,
+        full_table_name: str | FullyQualifiedName,
         schema: dict,
         primary_keys: t.Sequence[str],
         connection: sa.engine.Connection,
@@ -157,7 +160,7 @@ def prepare_table(  # type: ignore[override]  # noqa: PLR0913
 
     def copy_table_structure(
         self,
-        full_table_name: str,
+        full_table_name: str | FullyQualifiedName,
         from_table: sa.Table,
         connection: sa.engine.Connection,
         as_temp_table: bool = False,
@@ -203,13 +206,7 @@ def clone_table(  # noqa: PLR0913
         self, new_table_name, table, metadata, connection, temp_table
     ) -> sa.Table:
         """Clone a table."""
-        new_columns = [
-            sa.Column(
-                column.name,
-                column.type,
-            )
-            for column in table.columns
-        ]
+        new_columns = [sa.Column(column.name, column.type) for column in table.columns]
         if temp_table is True:
             new_table = sa.Table(
                 new_table_name, metadata, *new_columns, prefixes=["TEMPORARY"]
@@ -291,14 +288,34 @@ def pick_individual_type(self, jsonschema_type: dict):  # noqa: PLR0911
         if "object" in jsonschema_type["type"]:
             return JSONB()
         if "array" in jsonschema_type["type"]:
-            items_type = jsonschema_type.get("items")
-            if "string" == items_type:
-                return ARRAY(TEXT())
-            if "integer" == items_type:
-                return ARRAY(BIGINT())
-            else:
+            items = jsonschema_type.get("items")
+            # Case 1: items is a string
+            if isinstance(items, str):
+                return ARRAY(self.to_sql_type({"type": items}))
+
+            # Case 2: items are more complex
+            if isinstance(items, dict):
+                # Case 2.1: items are variants
+                if "type" not in items:
+                    return ARRAY(JSONB())
+
+                items_type = items["type"]
+
+                # Case 2.2: items are a single type
+                if isinstance(items_type, str):
+                    return ARRAY(self.to_sql_type({"type": items_type}))
+
+                # Case 2.3: items are a list of types
+                if isinstance(items_type, list):
+                    return ARRAY(self.to_sql_type({"type": items_type}))
+
+            # Case 3: tuples
+            if isinstance(items, list):
                 return ARRAY(JSONB())
 
+            # All other cases, return JSONB
+            return JSONB()
+
         # string formats
         if jsonschema_type.get("format") == "date-time":
             return TIMESTAMP()
@@ -404,7 +421,7 @@ def create_empty_table(  # type: ignore[override]  # noqa: PLR0913
 
     def prepare_column(  # noqa: PLR0913
         self,
-        full_table_name: str,
+        full_table_name: str | FullyQualifiedName,
         column_name: str,
         sql_type: sa.types.TypeEngine,
         connection: sa.engine.Connection | None = None,
@@ -822,7 +839,7 @@ def get_table_columns(  # type: ignore[override]
 
     def column_exists(  # type: ignore[override]
         self,
-        full_table_name: str,
+        full_table_name: str | FullyQualifiedName,
         column_name: str,
         connection: sa.engine.Connection,
     ) -> bool:
@@ -898,7 +915,7 @@ def process_bind_param(self, value, dialect):
             except ValueError as ex:
                 raise ValueError(f"Invalid hexadecimal string: {value}") from ex
 
-        if not isinstance(value, bytearray | memoryview | bytes):
+        if not isinstance(value, (bytearray, memoryview, bytes)):
             raise TypeError(
                 "HexByteString columns support only bytes or hex string values. "
                 f"{type(value)} is not supported"