Merge branch 'develop_v2' into lambert-conformal-conic

.github/workflows/frontend-gh-pages.yml

@@ -42,8 +42,8 @@ jobs:
         run: |
           touch .env
           echo VITE_APP_API_URL=${{ secrets.VITE_APP_API_URL }} >> .env
-          echo VITE_APP_URL=${{ secrets.VITE_APP_URL }} >> .env
-          echo VITE_OAUTH2_REDIRECT_URL=${{ secrets.VITE_OAUTH2_REDIRECT_URL }} >> .env
+          echo VITE_APP_FULL_URL=${{ secrets.VITE_APP_FULL_URL }} >> .env
+          echo VITE_APP_BASE=${{ secrets.VITE_APP_BASE }} >> .env
           cp .env ../.env
       - name: Install dependencies
         run: npm install

app/README.md

@@ -1,13 +1,35 @@
 # domain_subsetter_v2
 
-### Getting Started
+## Getting Started
+
+### Clone the repo, checkout this branch
+```console
+git clone https://github.com/CUAHSI/domain-subsetter.git
+cd domain-subsetter
+git checkout develop_v2
+```
+
+### API for local development
 ```console
+cd app
 cp .env.template .env
 make build
 make up
 ```
+The API will be available at http://0.0.0.0:8000
+
+### Frontend for local development
+```console
+cd app
+cp .env.template .env  #if you haven't already
+cd frontend
+npm install
+npm run dev
+```
+The frontend will be available at http://localhost:5173/domain-subsetter
+More detailed info is available in the [frontend readme](frontend/README.md)
 
-### Formatting
+## Formatting
 ```console
 make format
 ```

app/api/Dockerfile

@@ -7,9 +7,7 @@ RUN pip install -r requirements.txt
 COPY requirements-dev.txt requirements-dev.txt
 RUN pip install -r requirements-dev.txt
 
-RUN apt-get update
-RUN apt-get install -y wget
-RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc
+RUN python -m wget https://dl.min.io/client/mc/release/linux-amd64/mc
 RUN chmod +x mc
 RUN mv mc /usr/local/bin/mc
 

app/api/requirements.txt

@@ -8,4 +8,6 @@ uvicorn[standard]
 httpx_oauth
 minio
 pydantic-settings
-google-cloud-logging
+google-cloud-logging
+pyproj
+wget

app/api/subsetter/app/db.py

@@ -1,4 +1,5 @@
 from enum import Enum
+from functools import lru_cache
 from typing import List, Optional, Tuple
 
 import httpx
@@ -9,10 +10,14 @@
 
 from subsetter.config import get_settings
 
-DATABASE_URL = get_settings().mongo_url
-client = motor.motor_asyncio.AsyncIOMotorClient(DATABASE_URL, uuidRepresentation="standard")
+client = motor.motor_asyncio.AsyncIOMotorClient(get_settings().mongo_url, uuidRepresentation="standard")
 db = client[get_settings().mongo_database]
 
+client_hydroshare = motor.motor_asyncio.AsyncIOMotorClient(
+    get_settings().hydroshare_mongo_url, uuidRepresentation="standard"
+)
+db_hydroshare = client_hydroshare[get_settings().hydroshare_mongo_database]
+
 
 class OAuthAccount(BaseOAuthAccount):
     pass
@@ -33,14 +38,6 @@ class Submission(BaseModel):
     startedAt: Optional[str] = None
     finishedAt: Optional[str] = None
     estimatedDuration: Optional[int] = None
-    view_users: Optional[List[str]] = []
-
-    def add_user(self, username: str):
-        self.view_users.append(username)
-        self.view_users = list(set(self.view_users))
-
-    def remove_user(self, username: str):
-        self.view_users.remove(username)
 
 
 class User(BeanieBaseUser, Document):
@@ -55,10 +52,9 @@ async def update_profile(self):
         async def get_profile(token: str) -> Tuple[str, str]:
             async with httpx.AsyncClient() as client:
                 response = await client.get(
-                    "https://auth.cuahsi.io/realms/CUAHSI/protocol/openid-connect/userinfo",
+                    get_settings().user_info_endpoint,
                     headers={"Authorization": f"Bearer {token}"},
                 )
-
                 return response.json()
 
         profile = await get_profile(self.oauth_accounts[0].access_token)
@@ -89,3 +85,8 @@ async def update_submission(self, submission: Submission) -> None:
 
 async def get_user_db():
     yield BeanieUserDatabase(User, OAuthAccount)
+
+
+@lru_cache
+def get_hydroshare_access_db():
+    return db_hydroshare

app/api/subsetter/app/routers/access_control/policy_generation.py

@@ -1,61 +1,54 @@
-'''
-def admin_policy_attach(target, name):
-    arguments = ['mc', '--config-dir', '/hydroshare/', 'admin', 'policy', 'attach', target, name, '--user', name]
-    logger.info(arguments)
-    try:
-        _output = subprocess.check_output(arguments, user='hydro-service')
-        logger.info(_output)
-    except subprocess.CalledProcessError as e:
-        logger.exception(e.output)
-
+import copy
+import json
+import logging as logger
+import os
 import subprocess
+import tempfile
+from typing import Dict
+
+
 def admin_policy_create(target, name, file):
-    arguments = ['mc', '--config-dir', '/hydroshare/', '--json', 'admin', 'policy', 'create', target, name, file]
+    arguments = ['mc', '--json', 'admin', 'policy', 'create', target, name, file]
     logger.info(arguments)
     try:
-        _output = subprocess.check_output(arguments, user='hydro-service')
+        _output = subprocess.check_output(arguments)
         logger.info(_output)
     except subprocess.CalledProcessError as e:
         logger.exception(e.output)
-    admin_policy_attach(target, name)
+
 
 def admin_policy_remove(target, name):
-    arguments = ['mc', '--config-dir', '/hydroshare/', '--json', 'admin', 'policy', 'remove', target, name]
+    arguments = ['mc', '--json', 'admin', 'policy', 'rm', target, name]
     logger.info(arguments)
     try:
-        _output = subprocess.check_output(arguments, user='hydro-service')
+        _output = subprocess.check_output(arguments)
         logger.info(_output)
     except subprocess.CalledProcessError as e:
         logger.exception(e.output)
 
-def refresh_minio_policy(user):
-    policy = minio_policy(user)
-    logging.info(json.dumps(policy, indent=2))
-    if policy["Statement"]:
-        with tempfile.TemporaryDirectory(dir='/hs_tmp') as tmpdirname:
-            filepath = os.path.join(tmpdirname, "metadata.json")
-            fp = open(filepath, "w")
-            fp.write(json.dumps(policy))
-            fp.close()
-            admin_policy_remove(target='cuahsi', name=user.username)
-            admin_policy_create(target='cuahsi', name=user.username, file=filepath)
-    else:
-        admin_policy_remove(target='cuahsi', name=user.username)
-'''
 
-import copy
+def refresh_minio_policy(user, policy):
+    logger.info(json.dumps(policy, indent=2))
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        filepath = os.path.join(tmpdirname, "metadata.json")
+        fp = open(filepath, "w")
+        fp.write(json.dumps(policy))
+        fp.close()
+        admin_policy_create(target='cuahsi', name=user.username, file=filepath)
+    return policy
 
 
-def bucket_name(resource_id: str):
-    # raccess = ResourceAccess.objects.filter(resource__short_id=resource_id).first()
-    # return raccess.owners.first().username
-    return "subsetter-outputs"
-
-
-def create_view_statements(owner, submissions) -> list:
-    view_statement_template_get = {
+def create_view_statements(user, views: Dict[str, list[str]]) -> list:
+    if not views:
+        return []
+    view_statement_template_get_object = {
         "Effect": "Allow",
-        "Action": ["s3:GetBucketLocation", "s3:GetObject"],
+        "Action": ["s3:GetObject"],
+        "Resource": [],
+    }
+    view_statement_template_get_bucket = {
+        "Effect": "Allow",
+        "Action": ["s3:GetBucketLocation"],
         "Resource": [],
     }
     view_statement_template_listing = {
@@ -64,35 +57,53 @@ def create_view_statements(owner, submissions) -> list:
         "Resource": [],
         "Condition": {"StringLike": {"s3:prefix": []}},
     }
-    bucketname = bucket_name("blah")
-    get_resources = [f"arn:aws:s3:::{bucketname}/{owner.username}/*"]
-    view_statement = copy.deepcopy(view_statement_template_listing)
-    view_statement["Resource"] = [f"arn:aws:s3:::{bucketname}"]
-    view_statement["Condition"]["StringLike"]["s3:prefix"] = [f"{owner.username}/*"]
-    list_statements = [view_statement]
-    for user, submission in submissions:
-        bucketname = bucket_name(submission.workflow_id)
-        get_resources.append(
-            f"arn:aws:s3:::{bucketname}/{user.username}/{submission.workflow_name}/{submission.workflow_id}/*"
-        )
+
+    get_objectt_resources = []
+    get_bucket_resources = []
+    list_statements = []
+    for bucket_owner, resource_paths in views.items():
+        get_objectt_resources = get_objectt_resources + [
+            f"arn:aws:s3:::{bucket_owner}/{resource_path}/*" for resource_path in resource_paths
+        ]
+        get_bucket_resources.append(f"arn:aws:s3:::{bucket_owner}")
         view_statement = copy.deepcopy(view_statement_template_listing)
-        view_statement["Resource"] = [f"arn:aws:s3:::{bucketname}"]
+        view_statement["Resource"] = [f"arn:aws:s3:::{bucket_owner}"]
         view_statement["Condition"]["StringLike"]["s3:prefix"] = [
-            f"{user.username}/{submission.workflow_name}/{submission.workflow_id}/*"
+            f"{resource_path}/*" for resource_path in resource_paths
         ]
         list_statements.append(view_statement)
-    view_statement_template_get["Resource"] = get_resources
-    return list_statements + [view_statement_template_get]
+    view_statement_template_get_object["Resource"] = get_objectt_resources
+    view_statement_template_get_bucket["Resource"] = get_bucket_resources
+    return list_statements + [view_statement_template_get_object]
 
 
-# def create_edit_owner_statements(resource_ids: list[str]) -> list:
-#    edit_statement_template = {"Effect": "Allow", "Action": ["s3:*"], "Resource": []}
-#    edit_statement_template["Resource"] = [
-#        f"arn:aws:s3:::{bucket_name(resource_id)}/hydroshare/{resource_id}/*" for resource_id in resource_ids
-#    ]
-#    return [edit_statement_template]
+def create_edit_statements(user, edits: Dict[str, list[str]]) -> list:
+    edit_all_statement = {
+        "Effect": "Allow",
+        "Action": ["s3:*"],
+        "Resource": [f"arn:aws:s3:::{user.username}"],
+    }
+    edit_paths_resources = []
+    for bucket_owner, resource_paths in edits.items():
+        edit_paths_resources = edit_paths_resources + [
+            f"arn:aws:s3:::{bucket_owner}/{resource_path}/*" for resource_path in resource_paths
+        ]
+
+    edit_paths_statement = {
+        "Effect": "Allow",
+        "Action": ["s3:*Object"],
+        "Resource": edit_paths_resources,
+    }
+    list_bucket_statement = {
+        "Effect": "Allow",
+        "Action": ["s3:ListBucket"],
+        "Resource": [f"arn:aws:s3:::{bucket_owner}" for bucket_owner, _ in edits.items()],
+    }
+    statements = [edit_all_statement, edit_paths_statement, list_bucket_statement]
+    return [statement for statement in statements if statement]
 
 
-def minio_policy(user, view_submissions):
-    view_statements = create_view_statements(user, view_submissions)
-    return {"Version": "2012-10-17", "Statement": view_statements}
+def minio_policy(user, owners: Dict[str, list[str]], edits: Dict[str, list[str]], views: Dict[str, list[str]]):
+    statements = create_view_statements(user, views)
+    statements = statements + create_edit_statements(user, edits)
+    return {"Version": "2012-10-17", "Statement": statements}