Security features v2.6.2

.github/workflows/bandit.yml

@@ -28,6 +28,7 @@ jobs:
       - name: Install
         run: |
           python -m pip install --upgrade pip
+          python -m pip install typing-extensions
           python -m pip install bandit
       - name: Bandit
         run: bandit -r -lll -ii .

.github/workflows/checkov.yml

@@ -15,17 +15,56 @@ permissions:
 
 jobs:
   build:
-
+    env:
+      CDK_DEFAULT_REGION: us-east-1
+      AWS_REGION: us-east-1
+      CDK_DEFAULT_ACCOUNT: 111111111111
+      GITHUB_ACTIONS: true
+      CHECKOV_ACTIONS: true
+      DATAALL_REPO_BRANCH: main
+    strategy:
+      matrix:
+        python-version: [3.9]
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: dataall
+          POSTGRES_PASSWORD: docker
+          POSTGRES_USER: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
+      - name: Git clone
+        uses: actions/checkout@v4
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+      - name: Install CDK 
+        run: |
+          npm install -g aws-cdk cdk-nag
+          cdk --version
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: ${{ matrix.python-version }}
+      - name: Upgrade Pip
+        run: python -m pip install --upgrade pip
+      - name: Install Requirements
+        run: python -m pip install -r deploy/requirements.txt
+      - name: CDK Synth
+        run: |
+          npx cdk synth
+      - name: Resources CDK Synth
+        run: make checkov-synth
       - name: Test with Checkov
         id: checkov
-        uses: bridgecrewio/checkov-action@master
+        uses: bridgecrewio/checkov-action@v12.2845.0
         with:
           directory: .
           quiet: true

.github/workflows/snyk.yaml

@@ -0,0 +1,32 @@
+name: Snyk
+
+on:
+  workflow_dispatch:
+
+  schedule:
+    - cron: "0 9 * * 1"  # runs each Monday at 9:00 UTC
+
+permissions:
+  contents: read
+  security-events: write
+
+jobs:
+  security:
+    strategy:
+      matrix:
+        python-version: [3.9]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install All Requirements
+        run: make install
+      - name: Run Snyk to check for vulnerabilities
+        run: snyk test --all-projects --detection-depth=5 --severity-threshold=high
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          args: --all-projects --detection-depth=5 --severity-threshold=high

.gitignore

@@ -71,3 +71,4 @@ yarn-debug.log*
 yarn-error.log*
 .idea
 /.ruff_cache/
+/testdata.json

Makefile

@@ -16,7 +16,7 @@ venv:
 	@python3 -m venv "venv"
 	@/bin/bash -c "source venv/bin/activate"
 
-install: upgrade-pip install-deploy install-backend install-cdkproxy install-tests
+install: upgrade-pip install-deploy install-backend install-cdkproxy install-tests install-integration-tests install-custom-auth install-userguide
 
 upgrade-pip:
 	pip install --upgrade pip setuptools
@@ -36,6 +36,12 @@ install-tests:
 install-integration-tests:
 	pip install -r tests_new/integration_tests/requirements.txt
 
+install-custom-auth:
+	pip install -r deploy/custom_resources/custom_authorizer/requirements.txt
+
+install-userguide:
+	pip install -r documentation/userguide/requirements.txt
+
 lint:
 	pip install ruff
 	ruff check --fix
@@ -51,6 +57,10 @@ check-security: upgrade-pip install-backend install-cdkproxy
 	bandit -lll -r backend
 	safety check --ignore=51668,70612,70624
 
+checkov-synth: upgrade-pip install-backend install-cdkproxy install-tests
+	export PYTHONPATH=./backend:/./tests && \
+	python -m pytest -v -ra -k test_checkov tests
+
 test:
 	export PYTHONPATH=./backend:/./tests && \
 	python -m pytest -v -ra tests/
@@ -78,7 +88,7 @@ deploy-image:
 	docker push ${account}.dkr.ecr.${region}.amazonaws.com/${repo}:${image-tag}
 
 assume-role:
-	aws sts assume-role --role-arn "arn:aws:iam::${REMOTE_ACCOUNT_ID}:role/${REMOTE_ROLE}" --role-session-name "session1" >.assume_role_json
+	aws sts assume-role --role-arn "arn:aws:iam::${REMOTE_ACCOUNT_ID}:role/${REMOTE_ROLE}" --external-id ${EXTERNAL_ID} --role-session-name "session1" >.assume_role_json
 	echo "export AWS_ACCESS_KEY_ID=$$(cat .assume_role_json | jq '.Credentials.AccessKeyId' -r)" >.env.assumed_role
 	echo "export AWS_SECRET_ACCESS_KEY=$$(cat .assume_role_json | jq '.Credentials.SecretAccessKey' -r)" >>.env.assumed_role
 	echo "export AWS_SESSION_TOKEN=$$(cat .assume_role_json | jq '.Credentials.SessionToken' -r)" >>.env.assumed_role

backend/api_handler.py

@@ -15,13 +15,15 @@
     attach_tenant_policy_for_groups,
     check_reauth,
     validate_and_block_if_maintenance_window,
+    redact_creds,
 )
 from dataall.core.tasks.service_handlers import Worker
 from dataall.base.aws.sqs import SqsQueue
 from dataall.base.context import set_context, dispose_context, RequestContext
 from dataall.base.db import get_engine
 from dataall.base.loader import load_modules, ImportMode
 
+from graphql.pyutils import did_you_mean
 
 logger = logging.getLogger()
 logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO'))
@@ -31,11 +33,17 @@
 for name in ['boto3', 's3transfer', 'botocore', 'boto']:
     logging.getLogger(name).setLevel(logging.ERROR)
 
+ALLOW_INTROSPECTION = True if os.getenv('ALLOW_INTROSPECTION') == 'True' else False
+
+if not ALLOW_INTROSPECTION:
+    did_you_mean.__globals__['MAX_LENGTH'] = 0
+
 load_modules(modes={ImportMode.API})
 SCHEMA = bootstrap_schema()
 TYPE_DEFS = gql(SCHEMA.gql(with_directives=False))
 ENVNAME = os.getenv('envname', 'local')
 ENGINE = get_engine(envname=ENVNAME)
+ALLOWED_ORIGINS = os.getenv('ALLOWED_ORIGINS', '*')
 Worker.queue = SqsQueue.send
 
 
@@ -83,6 +91,7 @@ def handler(event, context):
         Return doc: https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html
     """
 
+    event = redact_creds(event)
     log.info('Lambda Event %s', event)
     log.debug('Env name %s', ENVNAME)
     log.debug('Engine %s', ENGINE.engine.url)
@@ -92,7 +101,7 @@ def handler(event, context):
             'statusCode': 200,
             'headers': {
                 'content-type': 'application/json',
-                'Access-Control-Allow-Origin': '*',
+                'Access-Control-Allow-Origin': ALLOWED_ORIGINS,
                 'Access-Control-Allow-Headers': '*',
                 'Access-Control-Allow-Methods': '*',
             },
@@ -135,18 +144,20 @@ def handler(event, context):
     else:
         raise Exception(f'Could not initialize user context from event {event}')
 
-    success, response = graphql_sync(schema=executable_schema, data=query, context_value=app_context)
+    success, response = graphql_sync(
+        schema=executable_schema, data=query, context_value=app_context, introspection=ALLOW_INTROSPECTION
+    )
 
     dispose_context()
     response = json.dumps(response)
 
-    log.info('Lambda Response %s', response)
-
+    log.info('Lambda Response Success: %s', success)
+    log.debug('Lambda Response %s', response)
     return {
         'statusCode': 200 if success else 400,
         'headers': {
             'content-type': 'application/json',
-            'Access-Control-Allow-Origin': '*',
+            'Access-Control-Allow-Origin': ALLOWED_ORIGINS,
             'Access-Control-Allow-Headers': '*',
             'Access-Control-Allow-Methods': '*',
         },

backend/aws_handler.py

@@ -7,7 +7,7 @@
 from dataall.base.loader import load_modules, ImportMode
 
 logger = logging.getLogger()
-logger.setLevel(os.environ.get('LOG_LEVEL'))
+logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO'))
 log = logging.getLogger(__name__)
 
 ENVNAME = os.getenv('envname', 'local')

backend/dataall/__init__.py

@@ -1,2 +1,13 @@
 from . import core, version
 from .base import utils, db, api
+import logging
+import os
+import sys
+
+logging.basicConfig(
+    level=os.environ.get('LOG_LEVEL', 'INFO'),
+    handlers=[logging.StreamHandler(sys.stdout)],
+    format='[%(levelname)s] %(message)s',
+)
+for name in ['boto3', 's3transfer', 'botocore', 'boto', 'urllib3']:
+    logging.getLogger(name).setLevel(logging.ERROR)

backend/dataall/base/api/__init__.py

@@ -12,6 +12,7 @@
 
 from dataall.base.api import gql
 from dataall.base.api.constants import GraphQLEnumMapper
+from dataall.base.api.queries import enumsQuery
 
 
 def bootstrap():

backend/dataall/base/api/constants.py

@@ -21,6 +21,14 @@ def to_label(cls, value):
                 return c.name
         return None
 
+    @classmethod
+    def has_value(cls, value):
+        return value in cls._value2member_map_
+
+    @classmethod
+    def has_key(cls, key):
+        return key in cls._member_map_
+
 
 class SortDirection(GraphQLEnumMapper):
     asc = 'asc'

backend/dataall/base/api/gql/graphql_type_modifiers.py

@@ -36,7 +36,7 @@ def gql(self):
             elif isinstance(self.of_type, Thunk):
                 return template(self.of_type.target.name)
             else:
-                raise Exception('Cant gql ')
+                raise Exception(f'Cant gql {self.of_type}')
 
     return Modifier
 

backend/dataall/base/api/queries.py

@@ -0,0 +1,11 @@
+from dataall.base.api import gql
+from dataall.base.api.resolvers import enum_resolver
+from dataall.base.api.types import EnumResult
+
+enumsQuery = gql.QueryField(
+    name='queryEnums',
+    args=[gql.Argument(name='enums_names', type=gql.ArrayType(gql.String))],
+    type=gql.ArrayType(EnumResult),
+    resolver=enum_resolver,
+    test_scope='Enums',
+)

backend/dataall/base/api/resolvers.py

@@ -0,0 +1,14 @@
+from dataall.base.api.constants import GraphQLEnumMapper
+
+
+def enum_resolver(context, source, enums_names):
+    result = []
+    for enum_class in GraphQLEnumMapper.__subclasses__():
+        if enum_class.__name__ in enums_names:
+            result.append(
+                {
+                    'name': enum_class.__name__,
+                    'items': [{'name': item.name, 'value': str(item.value)} for item in enum_class],
+                }
+            )
+    return result

backend/dataall/base/api/types.py

@@ -0,0 +1,17 @@
+from dataall.base.api import gql
+
+EnumItem = gql.ObjectType(
+    name='EnumItem',
+    fields=[
+        gql.Field(name='name', type=gql.String),
+        gql.Field(name='value', type=gql.String),
+    ],
+)
+
+EnumResult = gql.ObjectType(
+    name='EnumResult',
+    fields=[
+        gql.Field(name='name', type=gql.String),
+        gql.Field(name='items', type=gql.ArrayType(EnumItem)),
+    ],
+)

backend/dataall/base/aws/cognito.py

@@ -11,17 +11,22 @@ class Cognito(ServiceProvider):
     def __init__(self):
         self.client = boto3.client('cognito-idp', region_name=os.getenv('AWS_REGION', 'eu-west-1'))
 
+    def get_cognito_users(self, groupName):
+        envname = os.getenv('envname', 'local')
+        parameter_path = f'/dataall/{envname}/cognito/userpool'
+        ssm = boto3.client('ssm', region_name=os.getenv('AWS_REGION', 'eu-west-1'))
+        user_pool_id = ssm.get_parameter(Name=parameter_path)['Parameter']['Value']
+        paginator = self.client.get_paginator('list_users_in_group')
+        pages = paginator.paginate(UserPoolId=user_pool_id, GroupName=groupName)
+        cognito_user_list = []
+        for page in pages:
+            cognito_user_list += page['Users']
+
+        return cognito_user_list
+
     def get_user_emailids_from_group(self, groupName):
         try:
-            envname = os.getenv('envname', 'local')
-            parameter_path = f'/dataall/{envname}/cognito/userpool'
-            ssm = boto3.client('ssm', region_name=os.getenv('AWS_REGION', 'eu-west-1'))
-            user_pool_id = ssm.get_parameter(Name=parameter_path)['Parameter']['Value']
-            paginator = self.client.get_paginator('list_users_in_group')
-            pages = paginator.paginate(UserPoolId=user_pool_id, GroupName=groupName)
-            cognito_user_list = []
-            for page in pages:
-                cognito_user_list += page['Users']
+            cognito_user_list = self.get_cognito_users(groupName)
             group_email_ids = []
             attributes = []
             # Make a flat list
@@ -39,6 +44,20 @@ def get_user_emailids_from_group(self, groupName):
         else:
             return group_email_ids
 
+    def get_user_list_from_group(self, groupName):
+        try:
+            cognito_user_list = self.get_cognito_users(groupName)
+            group_usernames = [user['Username'] for user in cognito_user_list]
+        except Exception as e:
+            envname = os.getenv('envname', 'local')
+            if envname in ['local', 'dkrcompose']:
+                log.error('Local development environment does not support Cognito')
+                return ['[email protected]']
+            log.error(f'Failed to get usernames for Cognito group {groupName} due to {e}')
+            raise e
+        else:
+            return group_usernames
+
     def list_groups(self, envname: str, region: str):
         user_pool_id = None
         groups = []

backend/dataall/base/aws/quicksight.py

@@ -3,8 +3,7 @@
 
 from .sts import SessionHelper
 
-logger = logging.getLogger('QuicksightHandler')
-logger.setLevel(logging.DEBUG)
+logger = logging.getLogger(__name__)
 
 
 class QuicksightClient: