🌬️ Zephyr PoC

.github/dependabot.yml

@@ -30,6 +30,7 @@ updates:
       - "scripts/pagerduty/rota-to-slack"
       - "terraform/aws/analytical-platform-data-production/airflow/files/dev"
       - "terraform/aws/analytical-platform-data-production/airflow/files/prod"
+      - "terraform/aws/analytical-platform-development/zephyr-poc/src/airflow"
   - package-ecosystem: "terraform"
     schedule:
       interval: "daily"
@@ -69,6 +70,7 @@ updates:
       - "terraform/aws/analytical-platform-development/control-panel-message-broker"
       - "terraform/aws/analytical-platform-development/sagemaker"
       - "terraform/aws/analytical-platform-development/tooling-iam"
+      - "terraform/aws/analytical-platform-development/zephyr-poc"
       - "terraform/aws/analytical-platform-management-production/cluster"
       - "terraform/aws/analytical-platform-management-production/terraform-state"
       - "terraform/aws/analytical-platform-production/cluster"

terraform/aws/analytical-platform-development/zephyr-poc/acm.tf

@@ -0,0 +1,9 @@
+module "certificate" {
+  source  = "terraform-aws-modules/acm/aws"
+  version = "5.1.1"
+
+  zone_id     = data.aws_route53_zone.dev_analytical_platform_service_justice_gov_uk.zone_id
+  domain_name = local.mwaa_webserver_base_url
+
+  validation_method = "DNS"
+}

terraform/aws/analytical-platform-development/zephyr-poc/alb.tf

@@ -0,0 +1,81 @@
+module "alb" {
+  source  = "terraform-aws-modules/alb/aws"
+  version = "9.12.0"
+
+  name    = local.project_name
+  vpc_id  = module.vpc.vpc_id
+  subnets = module.vpc.public_subnets
+
+  security_group_ingress_rules = {
+    all_http = {
+      from_port   = 80
+      to_port     = 80
+      ip_protocol = "tcp"
+      description = "HTTP web traffic"
+      cidr_ipv4   = "0.0.0.0/0"
+    }
+    all_https = {
+      from_port   = 443
+      to_port     = 443
+      ip_protocol = "tcp"
+      description = "HTTPS web traffic"
+      cidr_ipv4   = "0.0.0.0/0"
+    }
+  }
+  security_group_egress_rules = {
+    all = {
+      ip_protocol = "-1"
+      cidr_ipv4   = module.vpc.vpc_cidr_block
+    }
+  }
+  listeners = {
+    ex-http-https-redirect = {
+      port     = 80
+      protocol = "HTTP"
+      redirect = {
+        port        = "443"
+        protocol    = "HTTPS"
+        status_code = "HTTP_301"
+      }
+    }
+    ex-https = {
+      port            = 443
+      protocol        = "HTTPS"
+      certificate_arn = module.certificate.acm_certificate_arn
+
+      forward = {
+        target_group_key = "ex-mwaa"
+      }
+    }
+  }
+  target_groups = {
+    ex-mwaa = {
+      name_prefix = "tg"
+      protocol    = "HTTPS"
+      port        = 443
+      target_type = "ip"
+      target_id   = "10.200.44.178"
+      health_check = {
+        enabled  = true
+        path     = "/"
+        port     = "traffic-port"
+        protocol = "HTTPS"
+        matcher  = "200,302"
+      }
+    }
+  }
+  additional_target_group_attachments = {
+    ex-mwaa = {
+      target_group_key = "ex-mwaa"
+      target_id        = "10.200.35.161"
+      port             = 443
+    }
+  }
+  # route53_records = {
+  #   zephyr = {
+  #     name    = "zephyr"
+  #     type    = "CNAME"
+  #     zone_id = data.aws_route53_zone.dev_analytical_platform_service_justice_gov_uk.zone_id
+  #   }
+  # }
+}

terraform/aws/analytical-platform-development/zephyr-poc/contrib/update-mwaa-environment.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+AWS_ACCOUNT_ID=${1}
+MWAA_ENVIRONMENT_NAME=${2}
+AWS_REGION=${3:-eu-west-2}
+AWS_ROLE=${4:-GlobalGitHubActionAdmin}
+
+assumeRole=$(aws sts assume-role \
+  --role-arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/${AWS_ROLE} \
+  --role-session-name analytical-platform-development-airflow)
+export assumeRole
+
+AWS_ACCESS_KEY_ID=$(echo ${assumeRole} | jq -r '.Credentials.AccessKeyId')
+export AWS_ACCESS_KEY_ID
+AWS_SECRET_ACCESS_KEY=$(echo ${assumeRole} | jq -r '.Credentials.SecretAccessKey')
+export AWS_SECRET_ACCESS_KEY
+AWS_SESSION_TOKEN=$(echo ${assumeRole} | jq -r '.Credentials.SessionToken')
+export AWS_SESSION_TOKEN
+
+aws --region "${AWS_REGION}" mwaa update-environment --name "${MWAA_ENVIRONMENT_NAME}"

terraform/aws/analytical-platform-development/zephyr-poc/data.tf

@@ -0,0 +1,40 @@
+data "aws_caller_identity" "session" {
+  provider = aws.session
+}
+
+data "aws_iam_session_context" "session" {
+  provider = aws.session
+
+  arn = data.aws_caller_identity.session.arn
+}
+
+data "aws_region" "current" {}
+
+data "aws_caller_identity" "current" {}
+
+data "aws_availability_zones" "available" {}
+
+data "aws_route53_zone" "dev_analytical_platform_service_justice_gov_uk" {
+  name = "dev.analytical-platform.service.justice.gov.uk"
+}
+
+# data "aws_vpc_endpoint" "mwaa_webserver" {
+#   service_name = aws_mwaa_environment.main.webserver_vpc_endpoint_service
+# }
+
+# data "aws_network_interface" "mwaa_webserver_vpce_network_interface_ids" {
+#   for_each = data.aws_vpc_endpoint.mwaa_webserver.network_interface_ids
+#   id       = each.value
+# }
+
+# output "mwaa_webserver_vpce_dns_entry" {
+#   value = data.aws_vpc_endpoint.mwaa_webserver.dns_entry
+# }
+
+# output "mwaa_webserver_vpce_network_interface_ids" {
+#   value = data.aws_vpc_endpoint.mwaa_webserver.network_interface_ids
+# }
+
+# output "mwaa_webserver_vpce_network_interface_private_ips" {
+#   value = [for i in data.aws_network_interface.mwaa_webserver_vpce_network_interface_ids : i.private_ip]
+# }

terraform/aws/analytical-platform-development/zephyr-poc/iam-policies.tf

@@ -0,0 +1,95 @@
+# Based on CMK policy from https://docs.aws.amazon.com/mwaa/latest/userguide/mwaa-create-role.html#mwaa-create-role-json
+data "aws_iam_policy_document" "airflow_execution_policy" {
+  statement {
+    effect  = "Deny"
+    actions = ["s3:ListAllMyBuckets"]
+    resources = [
+      "arn:aws:s3:::${local.bucket_name}",
+      "arn:aws:s3:::${local.bucket_name}/*"
+    ]
+  }
+  statement {
+    effect = "Allow"
+    actions = [
+      "s3:GetObject*",
+      "s3:GetBucket*",
+      "s3:List*"
+    ]
+    resources = [
+      "arn:aws:s3:::${local.bucket_name}",
+      "arn:aws:s3:::${local.bucket_name}/*"
+    ]
+  }
+  statement {
+    effect = "Allow"
+    actions = [
+      "logs:CreateLogStream",
+      "logs:CreateLogGroup",
+      "logs:PutLogEvents",
+      "logs:GetLogEvents",
+      "logs:GetLogRecord",
+      "logs:GetLogGroupFields",
+      "logs:GetQueryResults"
+    ]
+    resources = ["arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:airflow-${local.mwaa_environment_name}-*"]
+  }
+  statement {
+    effect    = "Allow"
+    actions   = ["logs:DescribeLogGroups"]
+    resources = ["*"]
+  }
+  statement {
+    effect    = "Allow"
+    actions   = ["s3:GetAccountPublicAccessBlock"]
+    resources = ["*"]
+  }
+  statement {
+    effect    = "Allow"
+    actions   = ["cloudwatch:PutMetricData"]
+    resources = ["*"]
+  }
+  statement {
+    effect = "Allow"
+    actions = [
+      "sqs:ChangeMessageVisibility",
+      "sqs:DeleteMessage",
+      "sqs:GetQueueAttributes",
+      "sqs:GetQueueUrl",
+      "sqs:ReceiveMessage",
+      "sqs:SendMessage"
+    ]
+    resources = ["arn:aws:sqs:${data.aws_region.current.name}:*:airflow-celery-*"]
+  }
+  statement {
+    effect = "Allow"
+    actions = [
+      "kms:Decrypt",
+      "kms:DescribeKey",
+      "kms:GenerateDataKey*",
+      "kms:Encrypt"
+    ]
+    resources = [module.airflow_kms.key_arn]
+    condition {
+      test     = "StringLike"
+      variable = "kms:ViaService"
+      values = [
+        "s3.${data.aws_region.current.name}.amazonaws.com",
+        "sqs.${data.aws_region.current.name}.amazonaws.com"
+      ]
+    }
+  }
+  # statement {
+  #   sid       = "AllowEKSDescribeCluster"
+  #   effect    = "Allow"
+  #   actions   = ["eks:DescribeCluster"]
+  #   resources = [module.eks.cluster_arn]
+  # }
+}
+
+module "airflow_execution_iam_policy" {
+  source  = "terraform-aws-modules/iam/aws//modules/iam-policy"
+  version = "5.48.0"
+
+  name   = local.execution_policy_name
+  policy = data.aws_iam_policy_document.airflow_execution_policy.json
+}

terraform/aws/analytical-platform-development/zephyr-poc/iam-roles.tf

@@ -0,0 +1,16 @@
+module "airflow_execution_iam_role" {
+  source  = "terraform-aws-modules/iam/aws//modules/iam-assumable-role"
+  version = "5.48.0"
+
+  create_role = true
+
+  role_name         = local.execution_role_name
+  role_requires_mfa = false
+
+  trusted_role_services = [
+    "airflow.amazonaws.com",
+    "airflow-env.amazonaws.com"
+  ]
+
+  custom_role_policy_arns = [module.airflow_execution_iam_policy.arn]
+}

terraform/aws/analytical-platform-development/zephyr-poc/kms.tf

@@ -0,0 +1,37 @@
+module "airflow_kms" {
+  source  = "terraform-aws-modules/kms/aws"
+  version = "3.1.1"
+
+  aliases               = [local.project_name]
+  enable_default_policy = true
+  key_statements = [
+    {
+      # https://docs.aws.amazon.com/mwaa/latest/userguide/custom-keys-certs.html#custom-keys-certs-grant-policies-attach
+      sid    = "AllowCloudWatchLogs"
+      effect = "Allow"
+      actions = [
+        "kms:Encrypt*",
+        "kms:Decrypt*",
+        "kms:ReEncrypt*",
+        "kms:GenerateDataKey*",
+        "kms:Describe*"
+      ]
+      resources = ["*"]
+      principals = [
+        {
+          type        = "Service"
+          identifiers = ["logs.${data.aws_region.current.name}.amazonaws.com"]
+        }
+      ]
+      conditions = [
+        {
+          test     = "ArnLike"
+          variable = "kms:EncryptionContext:aws:logs:arn"
+          values   = ["arn:aws:logs:${data.aws_region.current.name}:*:*"]
+        }
+      ]
+    }
+  ]
+
+  deletion_window_in_days = 7
+}

terraform/aws/analytical-platform-development/zephyr-poc/locals.tf

@@ -0,0 +1,9 @@
+locals {
+  project_name            = "mojap-zephyr-poc"
+  bucket_name             = "${local.project_name}-airflow"
+  execution_policy_name   = "${local.project_name}-airflow-execution"
+  execution_role_name     = "${local.project_name}-airflow-execution"
+  security_group_name     = "${local.project_name}-mwaa"
+  mwaa_environment_name   = local.project_name
+  mwaa_webserver_base_url = "zephyr.dev.analytical-platform.service.justice.gov.uk"
+}

terraform/aws/analytical-platform-development/zephyr-poc/main.tf

@@ -0,0 +1 @@
+