organize project for cloud builds

app/Makefile

@@ -1,6 +1,6 @@
 .DEFAULT_GOAL := all
-isort = isort /api
-black = black -S -l 120 --target-version py38 /api
+isort = isort /subsetter
+black = black -S -l 120 --target-version py310 /subsetter
 
 .PHONY: up
 up:
@@ -24,5 +24,5 @@ test:
 
 .PHONY: format
 format:
-	docker-compose run api "$(isort)"
-	docker-compose run api "$(black)"
+	docker-compose run api $(isort)
+	docker-compose run api $(black)

app/api/Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.10-slim
+
+COPY requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+# remove requirements-dev installation for deployments
+COPY requirements-dev.txt requirements-dev.txt
+RUN pip install -r requirements-dev.txt
+
+RUN apt-get update
+RUN apt-get install -y wget
+RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc
+RUN chmod +x mc
+RUN mv mc /usr/local/bin/mc
+
+RUN mc alias set cuahsi https://api.minio.cuahsi.io I7GKG576Xle9bsQZeUjY UyY08NY2GZ2ICAy5SC8TWb9KV80l8yCwfr8Dx58n
+
+COPY ./subsetter /subsetter
+
+ENV PYTHONPATH "/subsetter/:${PYTHONPATH}"
+
+EXPOSE 8000
+
+CMD uvicorn --host 0.0.0.0 --port 8000 --proxy-headers subsetter.main:app

app/api/app/db.py → app/api/subsetter/app/db.py

@@ -1,4 +1,3 @@
-import os
 from enum import Enum
 from typing import List, Optional, Tuple
 
@@ -8,9 +7,11 @@
 from fastapi_users.db import BaseOAuthAccount, BeanieBaseUser, BeanieUserDatabase
 from pydantic import BaseModel, Field
 
-DATABASE_URL = os.getenv("MONGO_URL")
+from subsetter.config import get_settings
+
+DATABASE_URL = get_settings().mongo_url
 client = motor.motor_asyncio.AsyncIOMotorClient(DATABASE_URL, uuidRepresentation="standard")
-db = client[os.getenv("MONGO_DATABASE")]
+db = client[get_settings().mongo_database]
 
 
 class OAuthAccount(BaseOAuthAccount):

app/api/app/models.py → app/api/subsetter/app/models.py

@@ -1,10 +1,11 @@
 from typing import Annotated
 
-from app.db import Submission, User
-from app.users import current_active_user
 from fastapi import Depends, HTTPException, Path, status
 from pydantic import BaseModel, Field
 
+from subsetter.app.db import Submission, User
+from subsetter.app.users import current_active_user
+
 
 class WorkflowParams(BaseModel):
     workflow_id: str = Field(title="Workflow ID", description="The id of the workflow")

app/api/subsetter/app/routers/access_control/__init__.py

@@ -0,0 +1 @@
+from subsetter.app.routers.access_control.router import router

app/api/app/routers/access_control/policy_generation.py → app/api/subsetter/app/routers/access_control/policy_generation.py

@@ -45,8 +45,6 @@ def refresh_minio_policy(user):
 
 import copy
 
-from app.db import Submission
-
 
 def bucket_name(resource_id: str):
     # raccess = ResourceAccess.objects.filter(resource__short_id=resource_id).first()

app/api/app/routers/access_control/router.py → app/api/subsetter/app/routers/access_control/router.py

@@ -1,10 +1,14 @@
-from app.db import Submission, User
-from app.models import WorkflowDep
-from app.users import current_active_user
-from fastapi import APIRouter, Depends, HTTPException, Query
+import json
+import os
+import subprocess
+import tempfile
+
+from fastapi import APIRouter, Depends, HTTPException
 from pydantic import BaseModel
 
-from .policy_generation import minio_policy
+from subsetter.app.db import Submission, User
+from subsetter.app.routers.access_control.policy_generation import minio_policy
+from subsetter.app.users import current_active_user
 
 router = APIRouter()
 
@@ -54,12 +58,6 @@ async def refresh_profile(user: User = Depends(current_active_user)):
     return user
 
 
-import json
-import os
-import subprocess
-import tempfile
-
-
 def admin_policy_create(name, policy, target="cuahsi"):
     with tempfile.TemporaryDirectory() as tmpdirname:
         print(policy)

app/api/subsetter/app/routers/argo/__init__.py

@@ -0,0 +1 @@
+from subsetter.app.routers.argo.router import router

app/api/app/routers/argo/router.py → app/api/subsetter/app/routers/argo/router.py

@@ -5,22 +5,23 @@
 
 import argo_workflows
 import google.cloud.logging as logging
-from app.users import current_active_user
 from argo_workflows.api import workflow_service_api
 from fastapi import APIRouter, Depends, Query
 
-from api.app.db import Submission, User
-from api.app.models import (
+from subsetter.app.db import Submission, User
+from subsetter.app.models import (
     LogsResponseModel,
     SubmissionResponseModel,
     UrlResponseModel,
     UserSubmissionsResponseModel,
     WorkflowDep,
 )
-from api.config import get_minio_client, get_settings
+from subsetter.app.users import current_active_user
+from subsetter.config import get_minio_client, get_settings
 
-logging_client = logging.Client()
-logging_client.setup_logging()
+if get_settings().cloud_run:
+    logging_client = logging.Client()
+    logging_client.setup_logging()
 
 router = APIRouter()
 

app/api/subsetter/app/routers/storage/__init__.py

@@ -0,0 +1 @@
+from subsetter.app.routers.storage.router import router

app/api/app/routers/storage/router.py → app/api/subsetter/app/routers/storage/router.py

@@ -1,14 +1,9 @@
-import json
-from typing import Annotated
+from fastapi import APIRouter, Depends
 
-from app.users import current_active_user
-from fastapi import APIRouter, Depends, Query
-
-from api.app.db import Submission, User
-
-# from .policy_generation import minio_policy
-from api.app.models import WorkflowDep
-from api.config import get_minio_client, get_settings
+from subsetter.app.db import Submission, User
+from subsetter.app.models import WorkflowDep
+from subsetter.app.users import current_active_user
+from subsetter.config import get_minio_client
 
 router = APIRouter()
 

app/api/app/users.py → app/api/subsetter/app/users.py

@@ -1,21 +1,18 @@
 import os
-from typing import Any, Dict, Optional, cast
+from typing import Any, Dict, Optional, Tuple, cast
 
-from app.db import User, get_user_db
+import httpx
 from beanie import PydanticObjectId
 from fastapi import Depends, Request
 from fastapi_users import BaseUserManager, FastAPIUsers
 from fastapi_users.authentication import AuthenticationBackend, BearerTransport, JWTStrategy
 from fastapi_users.db import BeanieUserDatabase, ObjectIDIDMixin
+from httpx_oauth.errors import GetIdEmailError
 from httpx_oauth.oauth2 import OAuth2
 
-SECRET = "SECRET"
-
-from typing import Any, Dict, Tuple, cast
+from subsetter.app.db import User, get_user_db
 
-import httpx
-from httpx_oauth.errors import GetIdEmailError
-from httpx_oauth.oauth2 import OAuth2
+SECRET = "SECRET"
 
 
 class CUAHSIOAuth2(OAuth2):

app/api/config/__init__.py → app/api/subsetter/config/__init__.py

@@ -24,6 +24,8 @@ class Settings(BaseSettings):
     minio_secret_key: str
     minio_api_url: str
 
+    cloud_run: bool = False
+
 
 @lru_cache()
 def get_settings() -> Settings:

app/api/main.py → app/api/subsetter/main.py

@@ -1,13 +1,13 @@
-from app.db import User, db
-from app.routers.access_control import router as access_control_router
-from app.routers.argo import router as argo_router
-from app.routers.storage import router as storage_router
-from app.schemas import UserRead, UserUpdate
-from app.users import SECRET, auth_backend, cuahsi_oauth_client, fastapi_users
 from beanie import init_beanie
 from fastapi import Depends, FastAPI
 
-from .config import get_settings
+from subsetter.app.db import User, db
+from subsetter.app.routers.access_control import router as access_control_router
+from subsetter.app.routers.argo import router as argo_router
+from subsetter.app.routers.storage import router as storage_router
+from subsetter.app.schemas import UserRead, UserUpdate
+from subsetter.app.users import SECRET, auth_backend, cuahsi_oauth_client, fastapi_users
+from subsetter.config import get_settings
 
 app = FastAPI()
 

app/docker-compose.yml

@@ -33,12 +33,12 @@ services:
       #- 5678:5678
     restart: unless-stopped
     volumes:
-      - ./api:/api
+      - ./api/subsetter:/subsetter
     build:
-      context: ./docker/api/
+      context: ./api/
       dockerfile: Dockerfile
     #command: ["pip install debugpy -t /tmp && python /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 -m uvicorn api.main:app --host 0.0.0.0 --port 8000"]
-    command: ["uvicorn api.main:app --host 0.0.0.0 --port 8000"]
+    #command: ["uvicorn subsetter.main:app --host 0.0.0.0 --port 8000"]
     env_file:
       - .env
     depends_on: