Adding AWS HealthOmics as a Module in "Play" tools

backend/dataall/base/cdkproxy/cdk.context.json

@@ -0,0 +1,52 @@
+{
+  "vpc-provider:account=856197974211:filter.isDefault=true:region=us-east-1:returnAsymmetricSubnets=true": {
+    "vpcId": "vpc-0bbcc12ab6cb2cc19",
+    "vpcCidrBlock": "172.31.0.0/16",
+    "ownerAccountId": "856197974211",
+    "availabilityZones": [],
+    "subnetGroups": [
+      {
+        "name": "Public",
+        "type": "Public",
+        "subnets": [
+          {
+            "subnetId": "subnet-0881ca6af336c69ca",
+            "cidr": "172.31.16.0/20",
+            "availabilityZone": "us-east-1a",
+            "routeTableId": "rtb-0e84eedc7b0edeb73"
+          },
+          {
+            "subnetId": "subnet-0b0dcf15be9546df8",
+            "cidr": "172.31.32.0/20",
+            "availabilityZone": "us-east-1b",
+            "routeTableId": "rtb-0e84eedc7b0edeb73"
+          },
+          {
+            "subnetId": "subnet-00518b260815523ec",
+            "cidr": "172.31.0.0/20",
+            "availabilityZone": "us-east-1c",
+            "routeTableId": "rtb-0e84eedc7b0edeb73"
+          },
+          {
+            "subnetId": "subnet-03d3d4329052576f6",
+            "cidr": "172.31.80.0/20",
+            "availabilityZone": "us-east-1d",
+            "routeTableId": "rtb-0e84eedc7b0edeb73"
+          },
+          {
+            "subnetId": "subnet-0abb0e50f5744a661",
+            "cidr": "172.31.48.0/20",
+            "availabilityZone": "us-east-1e",
+            "routeTableId": "rtb-0e84eedc7b0edeb73"
+          },
+          {
+            "subnetId": "subnet-08a2a4636d9f4c0b9",
+            "cidr": "172.31.64.0/20",
+            "availabilityZone": "us-east-1f",
+            "routeTableId": "rtb-0e84eedc7b0edeb73"
+          }
+        ]
+      }
+    ]
+  }
+}

backend/dataall/base/cdkproxy/requirements.txt

@@ -1,7 +1,7 @@
 aws-cdk-lib==2.99.0
-boto3==1.24.85
-boto3-stubs==1.24.85
-botocore==1.27.85
+boto3==1.28.23
+boto3-stubs==1.28.23
+botocore==1.31.23
 cdk-nag==2.7.2
 constructs==10.0.73
 starlette==0.27.0

backend/dataall/modules/catalog/api/registry.py

@@ -0,0 +1,60 @@
+from dataclasses import dataclass
+from typing import Type, Dict, Optional, Protocol, Union
+
+from dataall.base.api import gql
+from dataall.base.api.gql.graphql_union_type import UnionTypeRegistry
+from dataall.base.db import Resource
+from dataall.modules.catalog.indexers.base_indexer import BaseIndexer
+
+
+class Identifiable(Protocol):
+    @classmethod
+    def uri(cls):
+        ...
+
+
+@dataclass
+class GlossaryDefinition:
+    """Glossary's definition used for registration references of other modules"""
+    target_type: str
+    object_type: str
+    model: Union[Type[Resource], Identifiable]  # should be an intersection, but python typing doesn't have one yet
+    reindexer: Type[BaseIndexer] = None  # a callback to reindex glossaries in open search
+
+    def target_uri(self):
+        return self.model.uri()
+
+
+class GlossaryRegistry(UnionTypeRegistry):
+    """Registry of glossary definition and API to retrieve and reindex data"""
+    _DEFINITIONS: Dict[str, GlossaryDefinition] = {}
+
+    @classmethod
+    def register(cls, glossary: GlossaryDefinition) -> None:
+        cls._DEFINITIONS[glossary.target_type] = glossary
+
+    @classmethod
+    def find_model(cls, target_type: str) -> Optional[Resource]:
+        definition = cls._DEFINITIONS[target_type]
+        return definition.model if definition is not None else None
+
+    @classmethod
+    def find_object_type(cls, model: Resource) -> Optional[str]:
+        for _, definition in cls._DEFINITIONS.items():
+            if isinstance(model, definition.model):
+                return definition.object_type
+        return None
+
+    @classmethod
+    def definitions(cls):
+        return cls._DEFINITIONS.values()
+
+    @classmethod
+    def types(cls):
+        return [gql.Ref(definition.object_type) for definition in cls._DEFINITIONS.values()]
+
+    @classmethod
+    def reindex(cls, session, target_type: str, target_uri: str):
+        definition = cls._DEFINITIONS[target_type]
+        if definition.reindexer:
+            definition.reindexer.upsert(session, target_uri)

backend/dataall/modules/datapipelines/aws/codepipeline_datapipeline_client.py

@@ -0,0 +1,29 @@
+import logging
+
+from botocore.exceptions import ClientError
+
+from dataall.base.aws.sts import SessionHelper
+
+log = logging.getLogger('aws:codepipeline')
+
+
+class CodepipelineDatapipelineClient:
+    def __init__(self, aws_account_id, region) -> None:
+        self._aws_account_id = aws_account_id
+        self._region = region
+        self._session = SessionHelper.remote_session(aws_account_id)
+        self._client = self._session.client('codepipeline', region_name=region)
+
+    def get_pipeline_execution_summaries(self, codepipeline_name):
+        executions = []
+        try:
+            response = self._client.list_pipeline_executions(
+                pipelineName=codepipeline_name
+            )
+            executions = response['pipelineExecutionSummaries']
+        except ClientError as e:
+            log.warning(
+                f'Could not retrieve pipeline executions for {codepipeline_name} aws://{self._aws_account_id}:{self._region}'
+            )
+
+        return executions

backend/dataall/modules/datapipelines/aws/glue_datapipeline_client.py

@@ -0,0 +1,21 @@
+import logging
+
+from botocore.exceptions import ClientError
+
+from dataall.base.aws.sts import SessionHelper
+
+log = logging.getLogger('aws:glue')
+
+
+class GlueDatapipelineClient:
+    def __init__(self, aws_account_id, region) -> None:
+        self._session = SessionHelper.remote_session(aws_account_id)
+        self._client = self._session.client('glue', region_name=region)
+
+    def get_job_runs(self, datapipeline_job_name):
+        try:
+            response = self._client.get_job_runs(JobName=datapipeline_job_name)
+        except ClientError as e:
+            log.warning(f'Could not retrieve pipeline runs , {str(e)}')
+            return []
+        return response['JobRuns']

backend/dataall/modules/datapipelines/cdk/__init__.py

@@ -0,0 +1,13 @@
+from dataall.modules.datapipelines.cdk import datapipelines_cdk_cli_wrapper_extension, datapipelines_cdk_pipeline, \
+    env_role_datapipelines_stepfunctions_policy, env_role_datapipelines_lambda_policy, env_role_datapipelines_cicd_policy, datapipelines_pipeline, \
+    pivot_role_datapipelines_policy
+
+__all__ = [
+    'datapipelines_cdk_cli_wrapper_extension',
+    'datapipelines_cdk_pipeline',
+    'env_role_datapipelines_stepfunctions_policy',
+    'env_role_datapipelines_lambda_policy',
+    'env_role_datapipelines_cicd_policy',
+    'datapipelines_pipeline',
+    'pivot_role_datapipelines_policy'
+]

backend/dataall/modules/omics/__init__.py

@@ -0,0 +1,35 @@
+"""Contains the code related to X"""
+import logging
+from typing import Set
+
+from dataall.base.loader import ImportMode, ModuleInterface
+from dataall.core.environment.services.environment_resource_manager import EnvironmentResourceManager
+from dataall.modules.omics.db.omics_repository import OmicsRepository
+
+log = logging.getLogger(__name__)
+
+
+class OmicsApiModuleInterface(ModuleInterface):
+    """Implements ModuleInterface for omics GraphQl lambda"""
+
+    @staticmethod
+    def is_supported(modes: Set[ImportMode]) -> bool:
+        return ImportMode.API in modes
+
+    def __init__(self):
+        import dataall.modules.omics.api
+        from dataall.modules.omics.services.omics_permissions import GET_OMICS_RUN, UPDATE_OMICS_RUN
+
+        log.info("API of omics has been imported")
+
+
+class OmicsCdkModuleInterface(ModuleInterface):
+    """Implements ModuleInterface for omics ecs tasks"""
+
+    @staticmethod
+    def is_supported(modes: Set[ImportMode]) -> bool:
+        return ImportMode.CDK in modes
+
+    def __init__(self):
+        import dataall.modules.omics.cdk
+        log.info("API of Omics has been imported")

backend/dataall/modules/omics/api/__init__.py

@@ -0,0 +1,5 @@
+"""The package defines the schema for Omics Pipelines"""
+
+from dataall.modules.omics.api import input_types, mutations, queries, types, resolvers
+
+__all__ = ["types", "input_types", "queries", "mutations", "resolvers"]

backend/dataall/modules/omics/api/input_types.py

@@ -0,0 +1,37 @@
+"""The module defines GraphQL input types for Omics Runs"""
+
+from dataall.base.api import gql
+
+NewOmicsRunInput = gql.InputType(
+    name="NewOmicsRunInput",
+    arguments=[
+        gql.Field("environmentUri", type=gql.NonNullableType(gql.String)),
+        gql.Field("workflowId", type=gql.NonNullableType(gql.String)),
+        gql.Field("label", type=gql.NonNullableType(gql.String)),
+        gql.Field("destination", type=gql.String),
+        gql.Field("parameterTemplate", type=gql.String),
+        gql.Field("SamlAdminGroupName", type=gql.NonNullableType(gql.String)),
+    ],
+)
+
+# UpdateOmicsRunInput = gql.InputType(
+#     name="UpdateOmicsRunInput",
+#     arguments=[
+#         gql.Argument(name="label", type=gql.String),
+#         gql.Argument(name="description", type=gql.String),
+#         gql.Argument(name="tags", type=gql.ArrayType(gql.String)),
+#         gql.Argument(name="S3InputBucket", type=gql.String),
+#         gql.Argument(name="S3InputPrefix", type=gql.String),
+#         gql.Argument(name="S3OutputBucket", type=gql.String),
+#         gql.Argument(name="S3OutputPrefix", type=gql.String),
+#     ],
+# )
+
+OmicsFilter = gql.InputType(
+    name="OmicsFilter",
+    arguments=[
+        gql.Argument(name="term", type=gql.String),
+        gql.Argument(name="page", type=gql.Integer),
+        gql.Argument(name="pageSize", type=gql.Integer),
+    ],
+)

backend/dataall/modules/omics/api/mutations.py

@@ -0,0 +1,34 @@
+"""The module defines GraphQL mutations for Omics Pipelines"""
+
+from dataall.base.api import gql
+from .resolvers import *
+
+createOmicsRun = gql.MutationField(
+    name="createOmicsRun",
+    type=gql.Ref("OmicsRun"),
+    args=[gql.Argument(name="input", type=gql.NonNullableType(gql.Ref("NewOmicsRunInput")))],
+    resolver=create_omics_run,
+)
+# TODO implement updates to runs!
+# updateOmicsRun = gql.MutationField(
+#     name="updateOmicsRun",
+#     type=gql.Ref("OmicsRun"),
+#     args=[
+#         gql.Argument(name="OmicsRunUri", type=gql.NonNullableType(gql.String)),
+#         gql.Argument(name="input", type=gql.Ref("UpdateOmicsRunInput")),
+#     ],
+#     resolver=update_omics_pipeline,
+# )
+
+# TODO: not used in the frontend at the moment
+# deleteOmicsRun = gql.MutationField(
+#     name="deleteOmicsRun",
+#     type=gql.Boolean,
+#     args=[
+#         gql.Argument(name="runUri", type=gql.NonNullableType(gql.String)),
+#         gql.Argument(name="deleteFromAWS", type=gql.Boolean),
+#     ],
+#     resolver=delete_omics_run,
+# )
+
+

backend/dataall/modules/omics/api/queries.py

@@ -0,0 +1,25 @@
+"""The module defines GraphQL queries for Omics runs"""
+
+from dataall.base.api import gql
+from .resolvers import *
+
+listOmicsRuns = gql.QueryField(
+    name="listOmicsRuns",
+    args=[gql.Argument(name="filter", type=gql.Ref("OmicsFilter"))],
+    resolver=list_omics_runs,
+    type=gql.Ref("OmicsRunSearchResults"),
+)
+
+getOmicsWorkflow = gql.QueryField(
+    name="getOmicsWorkflow",
+    args=[gql.Argument(name="workflowId", type=gql.NonNullableType(gql.String))],
+    type=gql.Ref("OmicsWorkflow"),
+    resolver=get_omics_workflow,
+)
+
+listOmicsWorkflows = gql.QueryField(
+    name="listOmicsWorkflows",
+    args=[gql.Argument(name="filter", type=gql.Ref("OmicsFilter"))],
+    type=gql.Ref("OmicsWorkflows"),
+    resolver=list_omics_workflows,
+)