Merge branch 'database-transfer-updates'

* database-transfer-updates:
  New database transfer utilities
  Removed dev dependencies
  Updated lock files
  Update versions
  Updated version constraints
  Updatet transfer utilities
  Better stream utilities
  Added new utility functions
  Updated error management

app-frame/macrostrat/app_frame/exc.py

@@ -1,4 +1 @@
-class ApplicationError(Exception):
-    """Base class for all errors that should be caught and handled by the application."""
-
-    pass
+from macrostrat.utils import ApplicationError  # noqa

app-frame/pyproject.toml

@@ -6,19 +6,17 @@ packages = [
     { include = "macrostrat" },
     { include = "test_app" },
 ]
-version = "2.1.0"
+version = "2.2.0"
 
 [tool.poetry.dependencies]
-"macrostrat.utils" = "^1.1.0"
-python = "^3.8"
+"macrostrat.utils" = "^1.3.0"
+python = "^3.10"
 python-dotenv = "^1.0.0"
 toposort = "^1.5"
 rich = "^13"
 typer = "^0.12.5"
 packaging = "^24.1"
 
-[tool.poetry.dev-dependencies]
-
 [build-system]
 build-backend = "poetry.core.masonry.api"
 requires = ["poetry-core>=1.0.0"]

auth-system/pyproject.toml

@@ -1,13 +1,13 @@
 [tool.poetry]
 name = "macrostrat.auth_system"
-version = "1.0.2"
+version = "1.0.3"
 description = "Authentication system for Macrostrat and related apps"
 authors = ["Daven Quinn <[email protected]>"]
 readme = "README.md"
 packages = [{ include = "macrostrat" }]
 
 [tool.poetry.dependencies]
-python = "^3.9"
+python = "^3.10"
 "macrostrat.database" = "^3.3.1"
 "macrostrat.utils" = "^1.2.0"
 PyJWT = "^1.7.1 || ^2.0"

database/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## [3.5.0] - 2024-11-25
+
+- Add database transfer utilities for asynchronous `pg_load` and `pg_dump`
+  operations.
+
 ## [3.4.1] - 2024-10-28
 
 - Update the underlying version of `sqlparse` and `geoalchemy2`.

database/macrostrat/database/transfer/dump_database.py

@@ -1,13 +1,14 @@
 import asyncio
+import sys
 from pathlib import Path
 from typing import Optional
 
 import aiofiles
 from sqlalchemy.engine import Engine
 
 from macrostrat.utils import get_logger
-
-from .utils import _create_command, print_stdout, print_stream_progress
+from .stream_utils import print_stdout, print_stream_progress
+from .utils import _create_command
 
 log = get_logger(__name__)
 
@@ -45,11 +46,22 @@ async def pg_dump(
     )
 
 
-async def pg_dump_to_file(dumpfile: Path, *args, **kwargs):
-    proc = await pg_dump(*args, **kwargs)
+async def pg_dump_to_file(engine: Engine, dumpfile: Path | None, **kwargs):
+    proc = await pg_dump(engine, **kwargs)
+    if dumpfile is None or dumpfile == sys.stdout:
+        # If we have no dumpfile, just print to stdout
+        await _monitor_stdout(proc)
+        return
     # Open dump file as an async stream
     async with aiofiles.open(dumpfile, mode="wb") as dest:
         await asyncio.gather(
             asyncio.create_task(print_stream_progress(proc.stdout, dest)),
             asyncio.create_task(print_stdout(proc.stderr)),
         )
+
+
+async def _monitor_stdout(proc):
+    await asyncio.gather(
+        asyncio.create_task(print_stdout(proc.stdout)),
+        asyncio.create_task(print_stream_progress(proc.stderr, None)),
+    )

database/macrostrat/database/transfer/move_tables.py

@@ -3,10 +3,9 @@
 from sqlalchemy.engine import Engine
 
 from macrostrat.utils import get_logger
-
 from .dump_database import pg_dump
 from .restore_database import pg_restore
-from .utils import print_stdout, print_stream_progress
+from .stream_utils import print_stdout, print_stream_progress
 
 log = get_logger(__name__)
 

database/macrostrat/database/transfer/restore_database.py

@@ -7,13 +7,8 @@
 from sqlalchemy.engine import Engine
 
 from macrostrat.utils import get_logger
-
-from .utils import (
-    _create_command,
-    _create_database_if_not_exists,
-    print_stdout,
-    print_stream_progress,
-)
+from .stream_utils import print_stdout, print_stream_progress
+from .utils import _create_command, _create_database_if_not_exists
 
 console = Console()
 
@@ -56,11 +51,13 @@ async def pg_restore(
     )
 
 
-async def pg_restore_from_file(dumpfile: Path, *args, **kwargs):
-    proc = await pg_restore(*args, **kwargs)
+async def pg_restore_from_file(dumpfile: Path, engine: Engine, **kwargs):
+    proc = await pg_restore(engine, **kwargs)
     # Open dump file as an async stream
     async with aiofiles.open(dumpfile, mode="rb") as source:
         await asyncio.gather(
-            asyncio.create_task(print_stream_progress(source, proc.stdin)),
+            asyncio.create_task(
+                print_stream_progress(source, proc.stdin, prefix="Restored")
+            ),
             asyncio.create_task(print_stdout(proc.stderr)),
         )