Merge branch 'subsetter_argo_clean' into subsetter_front_clean

app/Makefile

@@ -1,6 +1,6 @@
 .DEFAULT_GOAL := all
-isort = isort /api
-black = black -S -l 120 --target-version py38 /api
+isort = isort /subsetter
+black = black -S -l 120 --target-version py310 /subsetter
 
 .PHONY: up
 up:
@@ -24,5 +24,5 @@ test:
 
 .PHONY: format
 format:
-	docker-compose run api "$(isort)"
-	docker-compose run api "$(black)"
+	docker-compose run api $(isort)
+	docker-compose run api $(black)

app/api/Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.10-slim
+
+COPY requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+# remove requirements-dev installation for deployments
+COPY requirements-dev.txt requirements-dev.txt
+RUN pip install -r requirements-dev.txt
+
+RUN apt-get update
+RUN apt-get install -y wget
+RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc
+RUN chmod +x mc
+RUN mv mc /usr/local/bin/mc
+
+COPY ./subsetter /subsetter
+
+ENV PYTHONPATH "/subsetter/:${PYTHONPATH}"
+
+EXPOSE 8000
+
+CMD uvicorn --host 0.0.0.0 --port 8000 --proxy-headers subsetter.main:app

app/docker/api/requirements.txt → app/api/requirements.txt

@@ -7,4 +7,5 @@ fastapi-users[beanie]
 uvicorn[standard]
 httpx_oauth
 minio
-pydantic-settings
+pydantic-settings
+google-cloud-logging

app/api/app/db.py → app/api/subsetter/app/db.py

@@ -1,4 +1,3 @@
-import os
 from enum import Enum
 from typing import List, Optional, Tuple
 
@@ -8,9 +7,11 @@
 from fastapi_users.db import BaseOAuthAccount, BeanieBaseUser, BeanieUserDatabase
 from pydantic import BaseModel, Field
 
-DATABASE_URL = os.getenv("MONGO_URL")
+from subsetter.config import get_settings
+
+DATABASE_URL = get_settings().mongo_url
 client = motor.motor_asyncio.AsyncIOMotorClient(DATABASE_URL, uuidRepresentation="standard")
-db = client[os.getenv("MONGO_DATABASE")]
+db = client[get_settings().mongo_database]
 
 
 class OAuthAccount(BaseOAuthAccount):

app/api/app/models.py → app/api/subsetter/app/models.py

@@ -1,10 +1,11 @@
 from typing import Annotated
 
-from app.db import Submission, User
-from app.users import current_active_user
 from fastapi import Depends, HTTPException, Path, status
 from pydantic import BaseModel, Field
 
+from subsetter.app.db import Submission, User
+from subsetter.app.users import current_active_user
+
 
 class WorkflowParams(BaseModel):
     workflow_id: str = Field(title="Workflow ID", description="The id of the workflow")

app/api/subsetter/app/routers/access_control/__init__.py

@@ -0,0 +1 @@
+from subsetter.app.routers.access_control.router import router

app/api/app/routers/access_control/policy_generation.py → app/api/subsetter/app/routers/access_control/policy_generation.py

@@ -45,8 +45,6 @@ def refresh_minio_policy(user):
 
 import copy
 
-from app.db import Submission
-
 
 def bucket_name(resource_id: str):
     # raccess = ResourceAccess.objects.filter(resource__short_id=resource_id).first()

app/api/app/routers/access_control/router.py → app/api/subsetter/app/routers/access_control/router.py

@@ -1,10 +1,14 @@
-from app.db import Submission, User
-from app.models import WorkflowDep
-from app.users import current_active_user
-from fastapi import APIRouter, Depends, HTTPException, Query
+import json
+import os
+import subprocess
+import tempfile
+
+from fastapi import APIRouter, Depends, HTTPException
 from pydantic import BaseModel
 
-from .policy_generation import minio_policy
+from subsetter.app.db import Submission, User
+from subsetter.app.routers.access_control.policy_generation import minio_policy
+from subsetter.app.users import current_active_user
 
 router = APIRouter()
 
@@ -54,12 +58,6 @@ async def refresh_profile(user: User = Depends(current_active_user)):
     return user
 
 
-import json
-import os
-import subprocess
-import tempfile
-
-
 def admin_policy_create(name, policy, target="cuahsi"):
     with tempfile.TemporaryDirectory() as tmpdirname:
         print(policy)

app/api/subsetter/app/routers/argo/__init__.py

@@ -0,0 +1 @@
+from subsetter.app.routers.argo.router import router

app/api/app/routers/argo/router.py → app/api/subsetter/app/routers/argo/router.py

@@ -1,21 +1,27 @@
 import json
+import logging as log
 import uuid
 from typing import Annotated
 
 import argo_workflows
-from app.users import current_active_user
+import google.cloud.logging as logging
 from argo_workflows.api import workflow_service_api
 from fastapi import APIRouter, Depends, Query
 
-from api.app.db import Submission, User
-from api.app.models import (
+from subsetter.app.db import Submission, User
+from subsetter.app.models import (
     LogsResponseModel,
     SubmissionResponseModel,
     UrlResponseModel,
     UserSubmissionsResponseModel,
     WorkflowDep,
 )
-from api.config import get_minio_client, get_settings
+from subsetter.app.users import current_active_user
+from subsetter.config import get_minio_client, get_settings
+
+if get_settings().cloud_run:
+    logging_client = logging.Client()
+    logging_client.setup_logging()
 
 router = APIRouter()
 
@@ -97,11 +103,12 @@ async def submit_parflow(
     hucs: Annotated[list[str] | None, Query()], user: User = Depends(current_active_user)
 ) -> SubmissionResponseModel:
     workflow_id = str(uuid.uuid4())
-    api_instance.submit_workflow(
+    api_response = api_instance.submit_workflow(
         namespace=get_settings().argo_namespace,
         body=parflow_submission_body(hucs, user.username, workflow_id),
         _preload_content=False,
     )
+    log.info(api_response.json())
     submission = Submission(workflow_id=workflow_id, workflow_name="parflow")
     return await upsert_submission(user, submission)
 
@@ -111,11 +118,12 @@ async def submit_nwm1(
     y_south: float, x_west: float, y_north: float, x_east: float, user: User = Depends(current_active_user)
 ) -> SubmissionResponseModel:
     workflow_id = str(uuid.uuid4())
-    api_instance.submit_workflow(
+    api_response = api_instance.submit_workflow(
         namespace=get_settings().argo_namespace,
         body=nwm1_submission_body(y_south, x_west, y_north, x_east, user.username, workflow_id),
         _preload_content=False,
     )
+    log.info(api_response.json())
     submission = Submission(workflow_id=workflow_id, workflow_name="nwm1")
     return await upsert_submission(user, submission)
 
@@ -125,11 +133,12 @@ async def submit_nwm2(
     y_south: float, x_west: float, y_north: float, x_east: float, user: User = Depends(current_active_user)
 ) -> SubmissionResponseModel:
     workflow_id = str(uuid.uuid4())
-    api_instance.submit_workflow(
+    api_response = api_instance.submit_workflow(
         namespace=get_settings().argo_namespace,
         body=nwm2_submission_body(y_south, x_west, y_north, x_east, user.username, workflow_id),
         _preload_content=False,
     )
+    log.info(api_response.json())
     submission = Submission(workflow_id=workflow_id, workflow_name="nwm2")
     return await upsert_submission(user, submission)
 
@@ -138,11 +147,14 @@ async def upsert_submission(user: User, submission: Submission) -> Submission:
     api_response = api_instance.get_workflow(
         namespace=get_settings().argo_namespace, name=submission.workflow_id, _preload_content=False
     )
+    log.info(api_response.json())
     status_json = api_response.json()["status"]
-    submission.phase = status_json["phase"]
+    if "phase" in status_json:
+        submission.phase = status_json["phase"]
+    if "estimatedDuration" in status_json:
+        submission.estimatedDuration = status_json["estimatedDuration"]
     submission.startedAt = status_json["startedAt"]
     submission.finishedAt = status_json["finishedAt"]
-    submission.estimatedDuration = status_json["estimatedDuration"]
     await user.update_submission(submission)
     return submission
 
@@ -192,6 +204,7 @@ async def logs(workflow_params: WorkflowDep) -> LogsResponseModel:
         log_options_container="main",
         _preload_content=False,
     )
+    log.info(api_response.json())
     return {"logs": parse_logs(api_response)}
 
 
@@ -210,6 +223,7 @@ async def argo_metadata(workflow_params: WorkflowDep):
     api_response = api_instance.get_workflow(
         namespace=get_settings().argo_namespace, name=workflow_params.workflow_id, _preload_content=False
     )
+    log.info(api_response.json())
     return {"metadata": api_response.json()["metadata"], "status": api_response.json()["status"]}
 
 

app/api/subsetter/app/routers/storage/__init__.py

@@ -0,0 +1 @@
+from subsetter.app.routers.storage.router import router

app/api/app/routers/storage/router.py → app/api/subsetter/app/routers/storage/router.py

@@ -1,14 +1,9 @@
-import json
-from typing import Annotated
+from fastapi import APIRouter, Depends
 
-from app.users import current_active_user
-from fastapi import APIRouter, Depends, Query
-
-from api.app.db import Submission, User
-
-# from .policy_generation import minio_policy
-from api.app.models import WorkflowDep
-from api.config import get_minio_client, get_settings
+from subsetter.app.db import Submission, User
+from subsetter.app.models import WorkflowDep
+from subsetter.app.users import current_active_user
+from subsetter.config import get_minio_client
 
 router = APIRouter()
 

app/api/app/users.py → app/api/subsetter/app/users.py

@@ -1,15 +1,15 @@
 import os
-import httpx
 from typing import Any, Dict, Optional, Tuple, cast
 
-from app.db import User, get_user_db
+import httpx
 from beanie import PydanticObjectId
 from fastapi import Depends, Request
 from fastapi_users import BaseUserManager, FastAPIUsers
 from fastapi_users.authentication import AuthenticationBackend, JWTStrategy, CookieTransport, BearerTransport
 from fastapi_users.db import BeanieUserDatabase, ObjectIDIDMixin
 from httpx_oauth.oauth2 import OAuth2, GetAccessTokenError, OAuth2Token
 from httpx_oauth.errors import GetIdEmailError
+from subsetter.app.db import User, get_user_db
 
 SECRET = "SECRET"