Tar a directory dump

Crown-Commercial-Service · Oct 11, 2024 · c76ba6f · c76ba6f
1 parent b5c20e8
commit c76ba6f
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/modules/postgres-etl/extract/extract_s3.tf b/modules/postgres-etl/extract/extract_s3.tf
@@ -13,7 +13,7 @@ resource "aws_s3_bucket_notification" "extract" {
   queue {
     queue_arn     = aws_sqs_queue.extract.arn
     events        = ["s3:ObjectCreated:*"]
-    filter_suffix = ".dump"
+    filter_suffix = ".tar"
   }
 }
 

diff --git a/modules/postgres-etl/extract/extract_task.tf b/modules/postgres-etl/extract/extract_task.tf
@@ -30,7 +30,7 @@ module "extract_task" {
       # N.B. $DUMP_FILENAME is injected by the Step Function task
       override_command = [
         "sh", "-c",
-        "apk upgrade && pg_dump -d $DB_CONNECTION_URL -Fc -j ${var.extract_task_pgrestore_workers} --no-acl --no-owner -f $MOUNT_POINT/$DUMP_FILENAME.dump && aws s3 cp --quiet $MOUNT_POINT/$DUMP_FILENAME.dump s3://${var.s3_extract_bucket_name}-${var.environment_name}/$DUMP_FILENAME-$(date +%Y-%m-%d-%H-%M-%S).dump && echo \"$DUMP_FILENAME successfully uploaded to S3\""
+        "apk upgrade && rm -rf $MOUNT_POINT/$DUMP_FILENAME && pg_dump -d $DB_CONNECTION_URL -Fd -j ${var.extract_task_pgrestore_workers} --no-acl --no-owner -f $MOUNT_POINT/$DUMP_FILENAME && tar -cf $MOUNT_POINT/$DUMP_FILENAME.tar -C $MOUNT_POINT/$DUMP_FILENAME && aws s3 cp --quiet $MOUNT_POINT/$DUMP_FILENAME.tar s3://${var.s3_extract_bucket_name}-${var.environment_name}/$DUMP_FILENAME-$(date +%Y-%m-%d-%H-%M-%S).tar && echo \"$DUMP_FILENAME successfully uploaded to S3\""
       ]
       port = null
       # ECS Execution role will need access to these

diff --git a/modules/postgres-etl/load/load_task.tf b/modules/postgres-etl/load/load_task.tf
@@ -26,7 +26,7 @@ module "load_task" {
       # N.B. $DUMP_FILENAME is injected by the Step Function task
       override_command = [
         "sh", "-c",
-        "aws s3 cp --quiet s3://${var.s3_load_bucket_name}-${var.environment_name}/$LOAD_FILENAME /mnt/efs0/etl-load.dump && echo \"$LOAD_FILENAME successfully downloaded from S3\" && pg_restore --clean --if-exists -d $DB_CONNECTION_URL -j ${var.load_task_pgrestore_workers} --no-acl --no-owner /mnt/efs0/etl-load.dump"
+        "aws s3 cp --quiet s3://${var.s3_load_bucket_name}-${var.environment_name}/$LOAD_FILENAME /mnt/efs0/etl-load.tar && echo \"$LOAD_FILENAME successfully downloaded from S3\" && rm -rf /mnt/efs0/etl-load && tar -xf /mnt/efs0/etl-load.tar -C /mnt/efs0 && pg_restore --clean --if-exists -d $DB_CONNECTION_URL -j ${var.load_task_pgrestore_workers} --no-acl --no-owner /mnt/efs0/etl-load"
       ]
       port = null
       secret_environment_variables = [