diff --git a/.github/workflows/google-cloudrun-source.yml b/.github/workflows/google-cloudrun-source.yml deleted file mode 100644 index 2f8bcdb6..00000000 --- a/.github/workflows/google-cloudrun-source.yml +++ /dev/null @@ -1,113 +0,0 @@ -# This workflow build and push a Docker container to Google Artifact Registry and deploy it on Cloud Run when a commit is pushed to the "master" branch -# -# Overview: -# -# 1. Authenticate to Google Cloud -# 2. Authenticate Docker to Artifact Registry -# 3. Build a docker container -# 4. Publish it to Google Artifact Registry -# 5. Deploy it to Cloud Run -# -# To configure this workflow: -# -# 1. Ensure the required Google Cloud APIs are enabled: -# -# Cloud Run run.googleapis.com -# Artifact Registry artifactregistry.googleapis.com -# -# 2. Create and configure Workload Identity Federation for GitHub (https://github.com/google-github-actions/auth#setting-up-workload-identity-federation) -# -# 3. Ensure the required IAM permissions are granted -# -# Cloud Run -# roles/run.admin -# roles/iam.serviceAccountUser (to act as the Cloud Run runtime service account) -# -# Artifact Registry -# roles/artifactregistry.admin (project or repository level) -# -# NOTE: You should always follow the principle of least privilege when assigning IAM roles -# -# 4. Create GitHub secrets for WIF_PROVIDER and WIF_SERVICE_ACCOUNT -# -# 5. Change the values for the GAR_LOCATION, SERVICE and REGION environment variables (below). -# -# NOTE: To use Google Container Registry instead, replace ${{ env.GAR_LOCATION }}-docker.pkg.dev with gcr.io -# -# For more support on how to run this workflow, please visit https://github.com/marketplace/actions/deploy-to-cloud-run -# -# Further reading: -# Cloud Run IAM permissions - https://cloud.google.com/run/docs/deploying -# Artifact Registry IAM permissions - https://cloud.google.com/artifact-registry/docs/access-control#roles -# Container Registry vs Artifact Registry - https://cloud.google.com/blog/products/application-development/understanding-artifact-registry-vs-container-registry -# Principle of least privilege - https://cloud.google.com/blog/products/identity-security/dont-get-pwned-practicing-the-principle-of-least-privilege - -name: Build and Deploy to Cloud Run - -on: - push: - branches: [ "subsetter_argo" ] - -env: - PROJECT_ID: apps-320517 - GAR_LOCATION: us-central1 - SERVICE: subsetter-api - REGION: us-central1 - -jobs: - deploy: - # Add 'id-token' with the intended permissions for workload identity federation - permissions: - contents: 'read' - id-token: 'write' - - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Google Auth - id: auth - uses: 'google-github-actions/auth@v0' - with: - token_format: 'access_token' - workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' # e.g. - projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider - service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' # e.g. - my-service-account@my-project.iam.gserviceaccount.com - - # NOTE: Alternative option - authentication via credentials json - # - name: Google Auth - # id: auth - # uses: 'google-github-actions/auth@v0' - # with: - # credentials_json: '${{ secrets.GCP_CREDENTIALS }}'' - - # BEGIN - Docker auth and build (NOTE: If you already have a container image, these Docker steps can be omitted) - - # Authenticate Docker to Google Cloud Artifact Registry - - name: Docker Auth - id: docker-auth - uses: 'docker/login-action@v1' - with: - username: 'oauth2accesstoken' - password: '${{ steps.auth.outputs.access_token }}' - registry: '${{ env.GAR_LOCATION }}-docker.pkg.dev' - - - name: Build and Push Container - run: |- - docker build -t "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.SERVICE }}:${{ github.sha }}" ./app/docker/api/ - docker push "${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.SERVICE }}:${{ github.sha }}" - - # END - Docker auth and build - - - name: Deploy to Cloud Run - id: deploy - uses: google-github-actions/deploy-cloudrun@v0 - with: - service: ${{ env.SERVICE }} - region: ${{ env.REGION }} - # NOTE: If using a pre-built image, update the image name here - image: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.SERVICE }}:${{ github.sha }} - - # If required, use the Cloud Run url output in later steps - - name: Show Output - run: echo ${{ steps.deploy.outputs.url }} diff --git a/app/Makefile b/app/Makefile index f3f5d423..8f505c10 100644 --- a/app/Makefile +++ b/app/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := all -isort = isort /api -black = black -S -l 120 --target-version py38 /api +isort = isort /subsetter +black = black -S -l 120 --target-version py310 /subsetter .PHONY: up up: @@ -24,5 +24,5 @@ test: .PHONY: format format: - docker-compose run api "$(isort)" - docker-compose run api "$(black)" + docker-compose run api $(isort) + docker-compose run api $(black) diff --git a/app/api/Dockerfile b/app/api/Dockerfile new file mode 100644 index 00000000..5f7e898c --- /dev/null +++ b/app/api/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.10-slim + +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt + +# remove requirements-dev installation for deployments +COPY requirements-dev.txt requirements-dev.txt +RUN pip install -r requirements-dev.txt + +RUN apt-get update +RUN apt-get install -y wget +RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc +RUN chmod +x mc +RUN mv mc /usr/local/bin/mc + +COPY ./subsetter /subsetter + +ENV PYTHONPATH "/subsetter/:${PYTHONPATH}" + +EXPOSE 8000 + +CMD uvicorn --host 0.0.0.0 --port 8000 --proxy-headers subsetter.main:app \ No newline at end of file diff --git a/app/api/app/routers/access_control/__init__.py b/app/api/app/routers/access_control/__init__.py deleted file mode 100644 index 23780433..00000000 --- a/app/api/app/routers/access_control/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .router import router diff --git a/app/api/app/routers/argo/__init__.py b/app/api/app/routers/argo/__init__.py deleted file mode 100644 index 23780433..00000000 --- a/app/api/app/routers/argo/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .router import router diff --git a/app/api/app/routers/storage/__init__.py b/app/api/app/routers/storage/__init__.py deleted file mode 100644 index 23780433..00000000 --- a/app/api/app/routers/storage/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .router import router diff --git a/app/docker/api/requirements-dev.txt b/app/api/requirements-dev.txt similarity index 100% rename from app/docker/api/requirements-dev.txt rename to app/api/requirements-dev.txt diff --git a/app/docker/api/requirements.txt b/app/api/requirements.txt similarity index 87% rename from app/docker/api/requirements.txt rename to app/api/requirements.txt index b3d2d4f3..51b2099e 100644 --- a/app/docker/api/requirements.txt +++ b/app/api/requirements.txt @@ -7,4 +7,5 @@ fastapi-users[beanie] uvicorn[standard] httpx_oauth minio -pydantic-settings \ No newline at end of file +pydantic-settings +google-cloud-logging \ No newline at end of file diff --git a/app/api/__init__.py b/app/api/subsetter/__init__.py similarity index 100% rename from app/api/__init__.py rename to app/api/subsetter/__init__.py diff --git a/app/api/app/db.py b/app/api/subsetter/app/db.py similarity index 95% rename from app/api/app/db.py rename to app/api/subsetter/app/db.py index 167942ec..ae2b9e28 100644 --- a/app/api/app/db.py +++ b/app/api/subsetter/app/db.py @@ -1,4 +1,3 @@ -import os from enum import Enum from typing import List, Optional, Tuple @@ -8,9 +7,11 @@ from fastapi_users.db import BaseOAuthAccount, BeanieBaseUser, BeanieUserDatabase from pydantic import BaseModel, Field -DATABASE_URL = os.getenv("MONGO_URL") +from subsetter.config import get_settings + +DATABASE_URL = get_settings().mongo_url client = motor.motor_asyncio.AsyncIOMotorClient(DATABASE_URL, uuidRepresentation="standard") -db = client[os.getenv("MONGO_DATABASE")] +db = client[get_settings().mongo_database] class OAuthAccount(BaseOAuthAccount): diff --git a/app/api/app/models.py b/app/api/subsetter/app/models.py similarity index 92% rename from app/api/app/models.py rename to app/api/subsetter/app/models.py index f237ab54..f3a5f83b 100644 --- a/app/api/app/models.py +++ b/app/api/subsetter/app/models.py @@ -1,10 +1,11 @@ from typing import Annotated -from app.db import Submission, User -from app.users import current_active_user from fastapi import Depends, HTTPException, Path, status from pydantic import BaseModel, Field +from subsetter.app.db import Submission, User +from subsetter.app.users import current_active_user + class WorkflowParams(BaseModel): workflow_id: str = Field(title="Workflow ID", description="The id of the workflow") diff --git a/app/api/subsetter/app/routers/access_control/__init__.py b/app/api/subsetter/app/routers/access_control/__init__.py new file mode 100644 index 00000000..e77a1878 --- /dev/null +++ b/app/api/subsetter/app/routers/access_control/__init__.py @@ -0,0 +1 @@ +from subsetter.app.routers.access_control.router import router diff --git a/app/api/app/routers/access_control/policy_generation.py b/app/api/subsetter/app/routers/access_control/policy_generation.py similarity index 99% rename from app/api/app/routers/access_control/policy_generation.py rename to app/api/subsetter/app/routers/access_control/policy_generation.py index 3a893a3b..861388ba 100644 --- a/app/api/app/routers/access_control/policy_generation.py +++ b/app/api/subsetter/app/routers/access_control/policy_generation.py @@ -45,8 +45,6 @@ def refresh_minio_policy(user): import copy -from app.db import Submission - def bucket_name(resource_id: str): # raccess = ResourceAccess.objects.filter(resource__short_id=resource_id).first() diff --git a/app/api/app/routers/access_control/router.py b/app/api/subsetter/app/routers/access_control/router.py similarity index 91% rename from app/api/app/routers/access_control/router.py rename to app/api/subsetter/app/routers/access_control/router.py index c375209b..a48c56ad 100644 --- a/app/api/app/routers/access_control/router.py +++ b/app/api/subsetter/app/routers/access_control/router.py @@ -1,10 +1,14 @@ -from app.db import Submission, User -from app.models import WorkflowDep -from app.users import current_active_user -from fastapi import APIRouter, Depends, HTTPException, Query +import json +import os +import subprocess +import tempfile + +from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel -from .policy_generation import minio_policy +from subsetter.app.db import Submission, User +from subsetter.app.routers.access_control.policy_generation import minio_policy +from subsetter.app.users import current_active_user router = APIRouter() @@ -54,12 +58,6 @@ async def refresh_profile(user: User = Depends(current_active_user)): return user -import json -import os -import subprocess -import tempfile - - def admin_policy_create(name, policy, target="cuahsi"): with tempfile.TemporaryDirectory() as tmpdirname: print(policy) diff --git a/app/api/subsetter/app/routers/argo/__init__.py b/app/api/subsetter/app/routers/argo/__init__.py new file mode 100644 index 00000000..346b1425 --- /dev/null +++ b/app/api/subsetter/app/routers/argo/__init__.py @@ -0,0 +1 @@ +from subsetter.app.routers.argo.router import router diff --git a/app/api/app/routers/argo/router.py b/app/api/subsetter/app/routers/argo/router.py similarity index 88% rename from app/api/app/routers/argo/router.py rename to app/api/subsetter/app/routers/argo/router.py index daacde7b..78feb924 100644 --- a/app/api/app/routers/argo/router.py +++ b/app/api/subsetter/app/routers/argo/router.py @@ -1,21 +1,27 @@ import json +import logging as log import uuid from typing import Annotated import argo_workflows -from app.users import current_active_user +import google.cloud.logging as logging from argo_workflows.api import workflow_service_api from fastapi import APIRouter, Depends, Query -from api.app.db import Submission, User -from api.app.models import ( +from subsetter.app.db import Submission, User +from subsetter.app.models import ( LogsResponseModel, SubmissionResponseModel, UrlResponseModel, UserSubmissionsResponseModel, WorkflowDep, ) -from api.config import get_minio_client, get_settings +from subsetter.app.users import current_active_user +from subsetter.config import get_minio_client, get_settings + +if get_settings().cloud_run: + logging_client = logging.Client() + logging_client.setup_logging() router = APIRouter() @@ -97,11 +103,12 @@ async def submit_parflow( hucs: Annotated[list[str] | None, Query()], user: User = Depends(current_active_user) ) -> SubmissionResponseModel: workflow_id = str(uuid.uuid4()) - api_instance.submit_workflow( + api_response = api_instance.submit_workflow( namespace=get_settings().argo_namespace, body=parflow_submission_body(hucs, user.username, workflow_id), _preload_content=False, ) + log.info(api_response.json()) submission = Submission(workflow_id=workflow_id, workflow_name="parflow") return await upsert_submission(user, submission) @@ -111,11 +118,12 @@ async def submit_nwm1( y_south: float, x_west: float, y_north: float, x_east: float, user: User = Depends(current_active_user) ) -> SubmissionResponseModel: workflow_id = str(uuid.uuid4()) - api_instance.submit_workflow( + api_response = api_instance.submit_workflow( namespace=get_settings().argo_namespace, body=nwm1_submission_body(y_south, x_west, y_north, x_east, user.username, workflow_id), _preload_content=False, ) + log.info(api_response.json()) submission = Submission(workflow_id=workflow_id, workflow_name="nwm1") return await upsert_submission(user, submission) @@ -125,11 +133,12 @@ async def submit_nwm2( y_south: float, x_west: float, y_north: float, x_east: float, user: User = Depends(current_active_user) ) -> SubmissionResponseModel: workflow_id = str(uuid.uuid4()) - api_instance.submit_workflow( + api_response = api_instance.submit_workflow( namespace=get_settings().argo_namespace, body=nwm2_submission_body(y_south, x_west, y_north, x_east, user.username, workflow_id), _preload_content=False, ) + log.info(api_response.json()) submission = Submission(workflow_id=workflow_id, workflow_name="nwm2") return await upsert_submission(user, submission) @@ -138,11 +147,14 @@ async def upsert_submission(user: User, submission: Submission) -> Submission: api_response = api_instance.get_workflow( namespace=get_settings().argo_namespace, name=submission.workflow_id, _preload_content=False ) + log.info(api_response.json()) status_json = api_response.json()["status"] - submission.phase = status_json["phase"] + if "phase" in status_json: + submission.phase = status_json["phase"] + if "estimatedDuration" in status_json: + submission.estimatedDuration = status_json["estimatedDuration"] submission.startedAt = status_json["startedAt"] submission.finishedAt = status_json["finishedAt"] - submission.estimatedDuration = status_json["estimatedDuration"] await user.update_submission(submission) return submission @@ -192,6 +204,7 @@ async def logs(workflow_params: WorkflowDep) -> LogsResponseModel: log_options_container="main", _preload_content=False, ) + log.info(api_response.json()) return {"logs": parse_logs(api_response)} @@ -210,6 +223,7 @@ async def argo_metadata(workflow_params: WorkflowDep): api_response = api_instance.get_workflow( namespace=get_settings().argo_namespace, name=workflow_params.workflow_id, _preload_content=False ) + log.info(api_response.json()) return {"metadata": api_response.json()["metadata"], "status": api_response.json()["status"]} diff --git a/app/api/subsetter/app/routers/storage/__init__.py b/app/api/subsetter/app/routers/storage/__init__.py new file mode 100644 index 00000000..c05399a8 --- /dev/null +++ b/app/api/subsetter/app/routers/storage/__init__.py @@ -0,0 +1 @@ +from subsetter.app.routers.storage.router import router diff --git a/app/api/app/routers/storage/router.py b/app/api/subsetter/app/routers/storage/router.py similarity index 86% rename from app/api/app/routers/storage/router.py rename to app/api/subsetter/app/routers/storage/router.py index d69c32ec..5b96268c 100644 --- a/app/api/app/routers/storage/router.py +++ b/app/api/subsetter/app/routers/storage/router.py @@ -1,14 +1,9 @@ -import json -from typing import Annotated +from fastapi import APIRouter, Depends -from app.users import current_active_user -from fastapi import APIRouter, Depends, Query - -from api.app.db import Submission, User - -# from .policy_generation import minio_policy -from api.app.models import WorkflowDep -from api.config import get_minio_client, get_settings +from subsetter.app.db import Submission, User +from subsetter.app.models import WorkflowDep +from subsetter.app.users import current_active_user +from subsetter.config import get_minio_client router = APIRouter() diff --git a/app/api/app/schemas.py b/app/api/subsetter/app/schemas.py similarity index 100% rename from app/api/app/schemas.py rename to app/api/subsetter/app/schemas.py diff --git a/app/api/app/users.py b/app/api/subsetter/app/users.py similarity index 98% rename from app/api/app/users.py rename to app/api/subsetter/app/users.py index b43fc16a..62157ded 100644 --- a/app/api/app/users.py +++ b/app/api/subsetter/app/users.py @@ -1,8 +1,7 @@ import os -import httpx from typing import Any, Dict, Optional, Tuple, cast -from app.db import User, get_user_db +import httpx from beanie import PydanticObjectId from fastapi import Depends, Request from fastapi_users import BaseUserManager, FastAPIUsers @@ -10,6 +9,7 @@ from fastapi_users.db import BeanieUserDatabase, ObjectIDIDMixin from httpx_oauth.oauth2 import OAuth2, GetAccessTokenError, OAuth2Token from httpx_oauth.errors import GetIdEmailError +from subsetter.app.db import User, get_user_db SECRET = "SECRET" diff --git a/app/api/config/__init__.py b/app/api/subsetter/config/__init__.py similarity index 93% rename from app/api/config/__init__.py rename to app/api/subsetter/config/__init__.py index fca76350..9b162561 100644 --- a/app/api/config/__init__.py +++ b/app/api/subsetter/config/__init__.py @@ -18,13 +18,13 @@ class Settings(BaseSettings): oauth2_client_id: str oauth2_client_secret: str + oauth2_redirect_url: str minio_access_key: str minio_secret_key: str minio_api_url: str - class Config: - env_file = ".env" + cloud_run: bool = False @lru_cache() diff --git a/app/api/main.py b/app/api/subsetter/main.py similarity index 66% rename from app/api/main.py rename to app/api/subsetter/main.py index 27db00c0..c82cf3da 100644 --- a/app/api/main.py +++ b/app/api/subsetter/main.py @@ -1,15 +1,18 @@ import os -import json -from app.db import User, db -from app.routers.access_control import router as access_control_router -from app.routers.argo import router as argo_router -from app.routers.storage import router as storage_router -from app.schemas import UserRead, UserUpdate -from app.users import SECRET, auth_backend, cookie_backend, cuahsi_oauth_client, front_oauth_client, fastapi_users +import subprocess + from beanie import init_beanie from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from subsetter.app.db import User, db +from subsetter.app.routers.access_control import router as access_control_router +from subsetter.app.routers.argo import router as argo_router +from subsetter.app.routers.storage import router as storage_router +from subsetter.app.schemas import UserRead, UserUpdate +from subsetter.app.users import SECRET, auth_backend, cuahsi_oauth_client, front_oauth_client, fastapi_users +from subsetter.config import get_settings + # TODO: get oauth working with swagger/redoc # Setting the base url for swagger docs # https://github.com/tiangolo/fastapi/pull/1547 @@ -32,7 +35,6 @@ allow_headers=["*"], ) - app.include_router( argo_router, # prefix="/auth/cuahsi", @@ -52,7 +54,9 @@ ) app.include_router( - fastapi_users.get_oauth_router(cuahsi_oauth_client, auth_backend, SECRET), + fastapi_users.get_oauth_router( + cuahsi_oauth_client, auth_backend, SECRET, redirect_url=get_settings().oauth2_redirect_url + ), prefix="/auth/cuahsi", tags=["auth"], ) @@ -84,3 +88,8 @@ async def on_startup(): User, ], ) + arguments = ['mc', 'alias', 'set', 'cuahsi', f"https://{get_settings().minio_api_url}", get_settings().minio_access_key, get_settings().minio_secret_key] + try: + _output = subprocess.check_output(arguments) + except subprocess.CalledProcessError as e: + raise diff --git a/app/docker-compose.yml b/app/docker-compose.yml index 158fd3fd..0b2b98a6 100644 --- a/app/docker-compose.yml +++ b/app/docker-compose.yml @@ -33,12 +33,11 @@ services: #- 5678:5678 restart: unless-stopped volumes: - - ./api:/api + - ./api/subsetter:/subsetter build: - context: ./docker/api/ + context: ./api/ dockerfile: Dockerfile #command: ["pip install debugpy -t /tmp && python /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 -m uvicorn api.main:app --host 0.0.0.0 --port 8000"] - command: ["uvicorn api.main:app --host 0.0.0.0 --port 8000"] env_file: - .env depends_on: diff --git a/app/docker/api/Dockerfile b/app/docker/api/Dockerfile deleted file mode 100644 index 49ef28af..00000000 --- a/app/docker/api/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM python:3.10-slim - -COPY requirements.txt requirements.txt -RUN pip install -r requirements.txt - -# remove requirements-dev installation for deployments -COPY requirements-dev.txt requirements-dev.txt -RUN pip install -r requirements-dev.txt - -ENV PYTHONPATH "${PYTHONPATH}:/api" - -EXPOSE 8000 - -ENTRYPOINT ["/bin/bash", "-c"] -CMD ["uvicorn", "main:app", "--host", "0.0.0.0"] \ No newline at end of file