From 7967ea89a6382ac2827760dd47523712f9ab6a14 Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Wed, 11 Dec 2024 00:51:10 +0000 Subject: [PATCH 1/2] mostly working Signed-off-by: Jacob Woffenden --- .../zephyr-poc/.terraform.lock.hcl | 44 +++++++++ .../zephyr-poc/acm.tf | 9 ++ .../zephyr-poc/alb.tf | 81 ++++++++++++++++ .../contrib/update-mwaa-environment.sh | 20 ++++ .../zephyr-poc/data.tf | 40 ++++++++ .../zephyr-poc/iam-policies.tf | 95 +++++++++++++++++++ .../zephyr-poc/iam-roles.tf | 16 ++++ .../zephyr-poc/kms.tf | 37 ++++++++ .../zephyr-poc/locals.tf | 9 ++ .../zephyr-poc/main.tf | 1 + .../zephyr-poc/mwaa.tf | 59 ++++++++++++ .../zephyr-poc/route53-records.tf | 15 +++ .../zephyr-poc/s3.tf | 51 ++++++++++ .../zephyr-poc/security-groups.tf | 19 ++++ .../airflow/dags/airflow_local_settings.py | 9 ++ .../zephyr-poc/src/airflow/requirements.txt | 3 + .../zephyr-poc/terraform.tf | 46 +++++++++ .../zephyr-poc/terraform.tfvars | 15 +++ .../zephyr-poc/variables.tf | 9 ++ .../zephyr-poc/vpc.tf | 17 ++++ 20 files changed, 595 insertions(+) create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/.terraform.lock.hcl create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/acm.tf create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/alb.tf create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/contrib/update-mwaa-environment.sh create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/data.tf create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/iam-policies.tf create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/iam-roles.tf create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/kms.tf create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/locals.tf create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/main.tf create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/mwaa.tf create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/route53-records.tf create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/s3.tf create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/security-groups.tf create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/src/airflow/dags/airflow_local_settings.py create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/src/airflow/requirements.txt create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/terraform.tf create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/terraform.tfvars create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/variables.tf create mode 100644 terraform/aws/analytical-platform-development/zephyr-poc/vpc.tf diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/.terraform.lock.hcl b/terraform/aws/analytical-platform-development/zephyr-poc/.terraform.lock.hcl new file mode 100644 index 0000000000..e1455796b5 --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/.terraform.lock.hcl @@ -0,0 +1,44 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.80.0" + constraints = ">= 3.29.0, >= 4.0.0, >= 4.40.0, >= 5.24.0, >= 5.37.0, >= 5.46.0, >= 5.49.0, >= 5.70.0, >= 5.73.0, 5.80.0" + hashes = [ + "h1:hhgPPhwxbuD3TaJq4clfKjy31vK68UGvN87PctNMuiY=", + "zh:0b1655e39639d60f2de2860a5df8642f9556ba0ca04529c1b861fde4935cb0df", + "zh:13dc0155e0a11edceee29ce687fc04c5a5a85f3324c67556472713cfd52e5807", + "zh:180f6cb2be44be14cfe329e0649121b774319f083b6e4e8fb749f85090d73121", + "zh:3158d44b74c67465f7f19f22c42b643840c8d18ce833e2ec86e8d93085b06926", + "zh:6351b5bf7cde5dc83e926944891570636069e05ca43341f4d1feda67773469bf", + "zh:6fa9db1532096ba50e842d369b6688979306d2295c7ead49b8a266b0d60962cc", + "zh:85d2fe75def7619ff2cc29102048875039cad088fafb62ecc14c3763e7b1e9d9", + "zh:9028d653f1d7341c6dfe2afe961b6541581e9043a474eac2faf90e6426a24f6d", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9c4e248c442bc60f07f9f089e5361f19936833370dc3c04b27916672b765f0e1", + "zh:a710a3979596e3f3938c3ec6bb748e604724d3a4afa96ed2c14f0a245cc41a11", + "zh:c27936bdf447779d0c0833bf52a9ef618985f5ea8e3e243d6266513520ca31c4", + "zh:c7681134a123486e72eaedc3f8d2d75e267dbbfd45fa7de5aea8f757af57f89b", + "zh:ea717ebad3561fd02591f9eecf30f3df5635405556fba2bdbf29fd42691bebac", + "zh:f4e1e8f23c58c3e8f4371f9c3379a723ab4155246e6b6daad8eb99e16666b2cb", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.3" + hashes = [ + "h1:obXguGZUWtNAO09f1f9Cb7hsPCOGXuGdN8bn/ohKRBQ=", + "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", + "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", + "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", + "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", + "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", + "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", + "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", + "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", + "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", + "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", + ] +} diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/acm.tf b/terraform/aws/analytical-platform-development/zephyr-poc/acm.tf new file mode 100644 index 0000000000..11ac40b637 --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/acm.tf @@ -0,0 +1,9 @@ +module "certificate" { + source = "terraform-aws-modules/acm/aws" + version = "5.1.1" + + zone_id = data.aws_route53_zone.dev_analytical_platform_service_justice_gov_uk.zone_id + domain_name = local.mwaa_webserver_base_url + + validation_method = "DNS" +} diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/alb.tf b/terraform/aws/analytical-platform-development/zephyr-poc/alb.tf new file mode 100644 index 0000000000..4fe618a359 --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/alb.tf @@ -0,0 +1,81 @@ +module "alb" { + source = "terraform-aws-modules/alb/aws" + version = "9.12.0" + + name = local.project_name + vpc_id = module.vpc.vpc_id + subnets = module.vpc.public_subnets + + security_group_ingress_rules = { + all_http = { + from_port = 80 + to_port = 80 + ip_protocol = "tcp" + description = "HTTP web traffic" + cidr_ipv4 = "0.0.0.0/0" + } + all_https = { + from_port = 443 + to_port = 443 + ip_protocol = "tcp" + description = "HTTPS web traffic" + cidr_ipv4 = "0.0.0.0/0" + } + } + security_group_egress_rules = { + all = { + ip_protocol = "-1" + cidr_ipv4 = module.vpc.vpc_cidr_block + } + } + listeners = { + ex-http-https-redirect = { + port = 80 + protocol = "HTTP" + redirect = { + port = "443" + protocol = "HTTPS" + status_code = "HTTP_301" + } + } + ex-https = { + port = 443 + protocol = "HTTPS" + certificate_arn = module.certificate.acm_certificate_arn + + forward = { + target_group_key = "ex-mwaa" + } + } + } + target_groups = { + ex-mwaa = { + name_prefix = "tg" + protocol = "HTTPS" + port = 443 + target_type = "ip" + target_id = "10.200.44.178" + health_check = { + enabled = true + path = "/" + port = "traffic-port" + protocol = "HTTPS" + matcher = "200,302" + } + } + } + additional_target_group_attachments = { + ex-mwaa = { + target_group_key = "ex-mwaa" + target_id = "10.200.35.161" + port = 443 + } + } + # route53_records = { + # zephyr = { + # name = "zephyr" + # type = "CNAME" + # zone_id = data.aws_route53_zone.dev_analytical_platform_service_justice_gov_uk.zone_id + # } + # } +} diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/contrib/update-mwaa-environment.sh b/terraform/aws/analytical-platform-development/zephyr-poc/contrib/update-mwaa-environment.sh new file mode 100644 index 0000000000..e5d04bc07a --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/contrib/update-mwaa-environment.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +AWS_ACCOUNT_ID=${1} +MWAA_ENVIRONMENT_NAME=${2} +AWS_REGION=${3:-eu-west-2} +AWS_ROLE=${4:-GlobalGitHubActionAdmin} + +assumeRole=$(aws sts assume-role \ + --role-arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/${AWS_ROLE} \ + --role-session-name analytical-platform-development-airflow) +export assumeRole + +AWS_ACCESS_KEY_ID=$(echo ${assumeRole} | jq -r '.Credentials.AccessKeyId') +export AWS_ACCESS_KEY_ID +AWS_SECRET_ACCESS_KEY=$(echo ${assumeRole} | jq -r '.Credentials.SecretAccessKey') +export AWS_SECRET_ACCESS_KEY +AWS_SESSION_TOKEN=$(echo ${assumeRole} | jq -r '.Credentials.SessionToken') +export AWS_SESSION_TOKEN + +aws --region "${AWS_REGION}" mwaa update-environment --name "${MWAA_ENVIRONMENT_NAME}" diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/data.tf b/terraform/aws/analytical-platform-development/zephyr-poc/data.tf new file mode 100644 index 0000000000..ba3fe81041 --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/data.tf @@ -0,0 +1,40 @@ +data "aws_caller_identity" "session" { + provider = aws.session +} + +data "aws_iam_session_context" "session" { + provider = aws.session + + arn = data.aws_caller_identity.session.arn +} + +data "aws_region" "current" {} + +data "aws_caller_identity" "current" {} + +data "aws_availability_zones" "available" {} + +data "aws_route53_zone" "dev_analytical_platform_service_justice_gov_uk" { + name = "dev.analytical-platform.service.justice.gov.uk" +} + +# data "aws_vpc_endpoint" "mwaa_webserver" { +# service_name = aws_mwaa_environment.main.webserver_vpc_endpoint_service +# } + +# data "aws_network_interface" "mwaa_webserver_vpce_network_interface_ids" { +# for_each = data.aws_vpc_endpoint.mwaa_webserver.network_interface_ids +# id = each.value +# } + +# output "mwaa_webserver_vpce_dns_entry" { +# value = data.aws_vpc_endpoint.mwaa_webserver.dns_entry +# } + +# output "mwaa_webserver_vpce_network_interface_ids" { +# value = data.aws_vpc_endpoint.mwaa_webserver.network_interface_ids +# } + +# output "mwaa_webserver_vpce_network_interface_private_ips" { +# value = [for i in data.aws_network_interface.mwaa_webserver_vpce_network_interface_ids : i.private_ip] +# } diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/iam-policies.tf b/terraform/aws/analytical-platform-development/zephyr-poc/iam-policies.tf new file mode 100644 index 0000000000..5f033ccaad --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/iam-policies.tf @@ -0,0 +1,95 @@ +# Based on CMK policy from https://docs.aws.amazon.com/mwaa/latest/userguide/mwaa-create-role.html#mwaa-create-role-json +data "aws_iam_policy_document" "airflow_execution_policy" { + statement { + effect = "Deny" + actions = ["s3:ListAllMyBuckets"] + resources = [ + "arn:aws:s3:::${local.bucket_name}", + "arn:aws:s3:::${local.bucket_name}/*" + ] + } + statement { + effect = "Allow" + actions = [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*" + ] + resources = [ + "arn:aws:s3:::${local.bucket_name}", + "arn:aws:s3:::${local.bucket_name}/*" + ] + } + statement { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:CreateLogGroup", + "logs:PutLogEvents", + "logs:GetLogEvents", + "logs:GetLogRecord", + "logs:GetLogGroupFields", + "logs:GetQueryResults" + ] + resources = ["arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:airflow-${local.mwaa_environment_name}-*"] + } + statement { + effect = "Allow" + actions = ["logs:DescribeLogGroups"] + resources = ["*"] + } + statement { + effect = "Allow" + actions = ["s3:GetAccountPublicAccessBlock"] + resources = ["*"] + } + statement { + effect = "Allow" + actions = ["cloudwatch:PutMetricData"] + resources = ["*"] + } + statement { + effect = "Allow" + actions = [ + "sqs:ChangeMessageVisibility", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + "sqs:SendMessage" + ] + resources = ["arn:aws:sqs:${data.aws_region.current.name}:*:airflow-celery-*"] + } + statement { + effect = "Allow" + actions = [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:GenerateDataKey*", + "kms:Encrypt" + ] + resources = [module.airflow_kms.key_arn] + condition { + test = "StringLike" + variable = "kms:ViaService" + values = [ + "s3.${data.aws_region.current.name}.amazonaws.com", + "sqs.${data.aws_region.current.name}.amazonaws.com" + ] + } + } + # statement { + # sid = "AllowEKSDescribeCluster" + # effect = "Allow" + # actions = ["eks:DescribeCluster"] + # resources = [module.eks.cluster_arn] + # } +} + +module "airflow_execution_iam_policy" { + source = "terraform-aws-modules/iam/aws//modules/iam-policy" + version = "5.48.0" + + name = local.execution_policy_name + policy = data.aws_iam_policy_document.airflow_execution_policy.json +} diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/iam-roles.tf b/terraform/aws/analytical-platform-development/zephyr-poc/iam-roles.tf new file mode 100644 index 0000000000..e23f396907 --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/iam-roles.tf @@ -0,0 +1,16 @@ +module "airflow_execution_iam_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role" + version = "5.48.0" + + create_role = true + + role_name = local.execution_role_name + role_requires_mfa = false + + trusted_role_services = [ + "airflow.amazonaws.com", + "airflow-env.amazonaws.com" + ] + + custom_role_policy_arns = [module.airflow_execution_iam_policy.arn] +} diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/kms.tf b/terraform/aws/analytical-platform-development/zephyr-poc/kms.tf new file mode 100644 index 0000000000..c013bae8e3 --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/kms.tf @@ -0,0 +1,37 @@ +module "airflow_kms" { + source = "terraform-aws-modules/kms/aws" + version = "3.1.1" + + aliases = [local.project_name] + enable_default_policy = true + key_statements = [ + { + # https://docs.aws.amazon.com/mwaa/latest/userguide/custom-keys-certs.html#custom-keys-certs-grant-policies-attach + sid = "AllowCloudWatchLogs" + effect = "Allow" + actions = [ + "kms:Encrypt*", + "kms:Decrypt*", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:Describe*" + ] + resources = ["*"] + principals = [ + { + type = "Service" + identifiers = ["logs.${data.aws_region.current.name}.amazonaws.com"] + } + ] + conditions = [ + { + test = "ArnLike" + variable = "kms:EncryptionContext:aws:logs:arn" + values = ["arn:aws:logs:${data.aws_region.current.name}:*:*"] + } + ] + } + ] + + deletion_window_in_days = 7 +} diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/locals.tf b/terraform/aws/analytical-platform-development/zephyr-poc/locals.tf new file mode 100644 index 0000000000..0cf1108d22 --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/locals.tf @@ -0,0 +1,9 @@ +locals { + project_name = "mojap-zephyr-poc" + bucket_name = "${local.project_name}-airflow" + execution_policy_name = "${local.project_name}-airflow-execution" + execution_role_name = "${local.project_name}-airflow-execution" + security_group_name = "${local.project_name}-mwaa" + mwaa_environment_name = local.project_name + mwaa_webserver_base_url = "zephyr.dev.analytical-platform.service.justice.gov.uk" +} diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/main.tf b/terraform/aws/analytical-platform-development/zephyr-poc/main.tf new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/main.tf @@ -0,0 +1 @@ + diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/mwaa.tf b/terraform/aws/analytical-platform-development/zephyr-poc/mwaa.tf new file mode 100644 index 0000000000..06069c6e9b --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/mwaa.tf @@ -0,0 +1,59 @@ +resource "aws_mwaa_environment" "main" { + name = local.mwaa_environment_name + airflow_version = "2.10.1" + environment_class = "mw1.medium" + weekly_maintenance_window_start = "SAT:00:00" + + execution_role_arn = module.airflow_execution_iam_role.iam_role_arn + + kms_key = module.airflow_kms.key_arn + + source_bucket_arn = module.airflow_bucket.s3_bucket_arn + dag_s3_path = "dags/" + requirements_s3_path = "requirements.txt" + requirements_s3_object_version = module.airflow_requirements_object.s3_object_version_id + + max_workers = 2 + min_workers = 1 + schedulers = 2 + + webserver_access_mode = "PRIVATE_ONLY" + + airflow_configuration_options = { + "webserver.warn_deployment_exposure" = 0 + "webserver.base_url" = local.mwaa_webserver_base_url + "webserver.instance_name" = "Zephyr PoC" + } + + network_configuration { + security_group_ids = [module.mwaa_security_group.security_group_id] + subnet_ids = slice(module.vpc.private_subnets, 0, 2) + } + + logging_configuration { + dag_processing_logs { + enabled = true + log_level = "INFO" + } + + scheduler_logs { + enabled = true + log_level = "INFO" + } + + task_logs { + enabled = true + log_level = "INFO" + } + + webserver_logs { + enabled = true + log_level = "INFO" + } + + worker_logs { + enabled = true + log_level = "INFO" + } + } +} diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/route53-records.tf b/terraform/aws/analytical-platform-development/zephyr-poc/route53-records.tf new file mode 100644 index 0000000000..c43642c7bf --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/route53-records.tf @@ -0,0 +1,15 @@ +module "route53_records" { + source = "terraform-aws-modules/route53/aws//modules/records" + version = "4.1.0" + + zone_id = data.aws_route53_zone.dev_analytical_platform_service_justice_gov_uk.zone_id + + records = [ + { + name = "zephyr" + type = "CNAME" + ttl = 300 + records = [module.alb.dns_name] + } + ] +} diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/s3.tf b/terraform/aws/analytical-platform-development/zephyr-poc/s3.tf new file mode 100644 index 0000000000..fce01d9174 --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/s3.tf @@ -0,0 +1,51 @@ +module "airflow_bucket" { + source = "terraform-aws-modules/s3-bucket/aws" + version = "4.2.2" + + bucket = local.bucket_name + + force_destroy = true + + versioning = { + enabled = true + } + + server_side_encryption_configuration = { + rule = { + bucket_key_enabled = true + apply_server_side_encryption_by_default = { + kms_master_key_id = module.airflow_kms.key_arn + sse_algorithm = "aws:kms" + } + } + } +} + +module "airflow_requirements_object" { + source = "terraform-aws-modules/s3-bucket/aws//modules/object" + version = "4.2.2" + + bucket = module.airflow_bucket.s3_bucket_id + key = "requirements.txt" + file_source = "src/airflow/requirements.txt" + force_destroy = true +} + +module "airflow_local_settings_object" { + source = "terraform-aws-modules/s3-bucket/aws//modules/object" + version = "4.2.2" + + bucket = module.airflow_bucket.s3_bucket_id + key = "dags/airflow_local_settings.py" + file_source = "src/airflow/dags/airflow_local_settings.py" + force_destroy = true +} + +resource "null_resource" "update_mwaa_environment" { + triggers = { + airflow_local_settings_object_version = module.airflow_local_settings_object.s3_object_version_id + } + provisioner "local-exec" { + command = "bash contrib/update-mwaa-environment.sh ${var.account_ids["analytical-platform-development"]} ${local.mwaa_environment_name}" + } +} diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/security-groups.tf b/terraform/aws/analytical-platform-development/zephyr-poc/security-groups.tf new file mode 100644 index 0000000000..4354858d6b --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/security-groups.tf @@ -0,0 +1,19 @@ +module "mwaa_security_group" { + source = "terraform-aws-modules/security-group/aws" + version = "5.2.0" + + name = local.security_group_name + vpc_id = module.vpc.vpc_id + + egress_cidr_blocks = ["0.0.0.0/0"] + egress_rules = ["all-all"] + + ingress_cidr_blocks = [module.vpc.vpc_cidr_block] + ingress_rules = ["all-all"] + + ingress_with_self = [ + { + rule = "all-all" + } + ] +} diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/src/airflow/dags/airflow_local_settings.py b/terraform/aws/analytical-platform-development/zephyr-poc/src/airflow/dags/airflow_local_settings.py new file mode 100644 index 0000000000..ccc39b6258 --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/src/airflow/dags/airflow_local_settings.py @@ -0,0 +1,9 @@ +from airflow.www.utils import UIAlert + +DASHBOARD_UIALERTS = [ + UIAlert( + 'This Airflow instance is a proof of concept and is not intended for production use.', + category="info", + html=True, + ) +] diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/src/airflow/requirements.txt b/terraform/aws/analytical-platform-development/zephyr-poc/src/airflow/requirements.txt new file mode 100644 index 0000000000..4d6767940a --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/src/airflow/requirements.txt @@ -0,0 +1,3 @@ +--constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.1/constraints-3.11.txt" +apache-airflow-providers-cncf-kubernetes==8.4.1 +kubernetes==30.1.0 diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/terraform.tf b/terraform/aws/analytical-platform-development/zephyr-poc/terraform.tf new file mode 100644 index 0000000000..6dfd62d635 --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/terraform.tf @@ -0,0 +1,46 @@ +terraform { + backend "s3" { + acl = "private" + bucket = "global-tf-state-aqsvzyd5u9" + encrypt = true + key = "aws/analytical-platform-development/zephry-poc/terraform.tfstate" + region = "eu-west-2" + dynamodb_table = "global-tf-state-aqsvzyd5u9-locks" + } + required_providers { + aws = { + source = "hashicorp/aws" + version = "5.80.0" + } + null = { + source = "hashicorp/null" + version = "3.2.3" + } + } + required_version = "~> 1.9" +} + +provider "aws" { + alias = "session" +} + +provider "aws" { + region = "eu-west-2" + assume_role { + role_arn = "arn:aws:iam::${var.account_ids["analytical-platform-development"]}:role/GlobalGitHubActionAdmin" + } + default_tags { + tags = var.tags + } +} + +provider "aws" { + alias = "analytical-platform-management-production" + region = "eu-west-2" + assume_role { + role_arn = can(regex("AdministratorAccess", data.aws_iam_session_context.session.issuer_arn)) ? null : "arn:aws:iam::${var.account_ids["analytical-platform-management-production"]}:role/GlobalGitHubActionAdmin" + } + default_tags { + tags = var.tags + } +} diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/terraform.tfvars b/terraform/aws/analytical-platform-development/zephyr-poc/terraform.tfvars new file mode 100644 index 0000000000..b2b64928e9 --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/terraform.tfvars @@ -0,0 +1,15 @@ +account_ids = { + analytical-platform-development = "525294151996" + analytical-platform-management-production = "042130406152" +} + +tags = { + business-unit = "Platforms" + application = "Analytical Platform" + component = "Zephyr Proof of Concept" + environment = "development" + is-production = "false" + owner = "analytical-platform:analytical-platform@digital.justice.gov.uk" + infrastructure-support = "analytical-platform:analytical-platform@digital.justice.gov.uk" + source-code = "github.com/ministryofjustice/analytical-platform/terraform/aws/analytical-platform-development/zephyr-poc" +} diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/variables.tf b/terraform/aws/analytical-platform-development/zephyr-poc/variables.tf new file mode 100644 index 0000000000..05c621dfe1 --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/variables.tf @@ -0,0 +1,9 @@ +variable "account_ids" { + type = map(string) + description = "Map of account names to account IDs" +} + +variable "tags" { + type = map(string) + description = "Map of tags to apply to resources" +} diff --git a/terraform/aws/analytical-platform-development/zephyr-poc/vpc.tf b/terraform/aws/analytical-platform-development/zephyr-poc/vpc.tf new file mode 100644 index 0000000000..c1a9766946 --- /dev/null +++ b/terraform/aws/analytical-platform-development/zephyr-poc/vpc.tf @@ -0,0 +1,17 @@ +/* VPC already exists in APC, this is a dummy */ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "5.16.0" + + name = local.project_name + azs = slice(data.aws_availability_zones.available.names, 0, 3) + cidr = "10.200.0.0/18" + public_subnets = ["10.200.0.0/27", "10.200.0.32/27", "10.200.0.64/27"] + private_subnets = ["10.200.32.0/21", "10.200.40.0/21", "10.200.48.0/21"] + + enable_nat_gateway = true + one_nat_gateway_per_az = false + single_nat_gateway = true + + enable_flow_log = false +} From dc54efeb18056369d0e0837477c06203b72102d9 Mon Sep 17 00:00:00 2001 From: Ministry of Justice Data Platform Robot <125977389+moj-data-platform-robot@users.noreply.github.com> Date: Wed, 11 Dec 2024 00:53:23 +0000 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=A4=96=20Update=20.github/dependabot.?= =?UTF-8?q?yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/dependabot.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0ce2cf9600..4d447b2bee 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -30,6 +30,7 @@ updates: - "scripts/pagerduty/rota-to-slack" - "terraform/aws/analytical-platform-data-production/airflow/files/dev" - "terraform/aws/analytical-platform-data-production/airflow/files/prod" + - "terraform/aws/analytical-platform-development/zephyr-poc/src/airflow" - package-ecosystem: "terraform" schedule: interval: "daily" @@ -69,6 +70,7 @@ updates: - "terraform/aws/analytical-platform-development/control-panel-message-broker" - "terraform/aws/analytical-platform-development/sagemaker" - "terraform/aws/analytical-platform-development/tooling-iam" + - "terraform/aws/analytical-platform-development/zephyr-poc" - "terraform/aws/analytical-platform-management-production/cluster" - "terraform/aws/analytical-platform-management-production/terraform-state" - "terraform/aws/analytical-platform-production/cluster"