redpanda-data · gene-redpanda · Oct 11, 2023 · Sep 20, 2023 · Sep 22, 2023 · Sep 22, 2023
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -2,11 +2,27 @@ agents:
   queue: "k8s-builders"
 
 steps:
+  - label: gcp ubuntu
+    key: tiered-up-gcp-ubuntu
+    command: ./.buildkite/scripts/standup-gcp-cluster.sh --tf_dir "gcp" --image "ubuntu-os-cloud/ubuntu-2204-lts" --prefix "ci-ub-ts" --cluster_type "tiered-storage-cluster" --gcp_creds "$DEVEX_GCP_CREDS_BASE64"
+    plugins:
+      - docker#v5.8.0:
+          image: glrp/atgt:latest
+          environment:
+            - DEVEX_GCP_CREDS_BASE64
+  - label: gcp fedora
+    key: tiered-up-gcp-fedora
+    command: ./.buildkite/scripts/standup-gcp-cluster.sh --tf_dir "gcp" --image "fedora-cloud/fedora-cloud-37" --prefix "ci-fd-ts" --cluster_type "tiered-storage-cluster" --gcp_creds "$DEVEX_GCP_CREDS_BASE64"
+    plugins:
+      - docker#v5.8.0:
+          image: glrp/atgt:latest
+          environment:
+            - DEVEX_GCP_CREDS_BASE64
   - label: test basic cluster standup ubuntu
     key: basic-up-ubuntu
     command: .buildkite/scripts/standup-cluster.sh --prefix=rp-basic-ub --distro=ubuntu-focal --tiered=false --unstable=false --taskname=basic-cluster
     plugins:
-      - docker#v5.4.0:
+      - docker#v5.8.0:
           image: glrp/atgt:latest
           environment:
             - DA_AWS_ACCESS_KEY_ID
@@ -16,7 +32,7 @@ steps:
     key: tiered-up-ubuntu
     command: .buildkite/scripts/standup-cluster.sh --prefix=rp-tier-ub --distro=ubuntu-focal --tiered=true --unstable=false --taskname=tiered-storage-cluster
     plugins:
-      - docker#v5.4.0:
+      - docker#v5.8.0:
           image: glrp/atgt:latest
           environment:
             - DA_AWS_ACCESS_KEY_ID
@@ -26,7 +42,7 @@ steps:
     key: tiered-up-unstable-ubuntu
     command: .buildkite/scripts/standup-cluster.sh --prefix=rp-un-tier-ub --distro=ubuntu-focal --tiered=true --unstable=true --taskname=tiered-storage-cluster
     plugins:
-      - docker#v5.4.0:
+      - docker#v5.8.0:
           image: glrp/atgt:latest
           environment:
             - DA_AWS_ACCESS_KEY_ID
@@ -36,7 +52,7 @@ steps:
     key: tiered-up-fedora
     command: .buildkite/scripts/standup-cluster.sh --prefix=rp-tier-fd --distro=Fedora-Cloud-Base-36 --tiered=true --unstable=false --taskname=tiered-storage-cluster
     plugins:
-      - docker#v5.4.0:
+      - docker#v5.8.0:
           image: glrp/atgt:latest
           environment:
             - DA_AWS_ACCESS_KEY_ID
@@ -46,7 +62,7 @@ steps:
     key: tiered-up-unstable-fedora
     command: .buildkite/scripts/standup-cluster.sh --prefix=rp-un-tier-fd --distro=Fedora-Cloud-Base-36 --tiered=true --unstable=true --taskname=tiered-storage-cluster
     plugins:
-      - docker#v5.4.0:
+      - docker#v5.8.0:
           image: glrp/atgt:latest
           environment:
             - DA_AWS_ACCESS_KEY_ID
@@ -56,7 +72,7 @@ steps:
     key: tiered-up-ubuntu-is4
     command: .buildkite/scripts/standup-cluster.sh --prefix=rp-tier-ub-is --distro=ubuntu-focal --tiered=true --unstable=false --taskname=tiered-storage-cluster --machinearch=arm64 --instancetype=is4gen.4xlarge
     plugins:
-      - docker#v5.4.0:
+      - docker#v5.8.0:
           image: glrp/atgt:latest
           environment:
             - DA_AWS_ACCESS_KEY_ID
@@ -66,7 +82,7 @@ steps:
     key: tiered-up-unstable-ubuntu-is4
     command: .buildkite/scripts/standup-cluster.sh --prefix=rp-un-tier-ub-is --distro=ubuntu-focal --tiered=true --unstable=true --taskname=tiered-storage-cluster --machinearch=arm64 --instancetype=is4gen.4xlarge
     plugins:
-      - docker#v5.4.0:
+      - docker#v5.8.0:
           image: glrp/atgt:latest
           environment:
             - DA_AWS_ACCESS_KEY_ID
@@ -76,7 +92,7 @@ steps:
     key: tiered-up-fedora-is4
     command: .buildkite/scripts/standup-cluster.sh --prefix=rp-tier-fd-is --distro=Fedora-Cloud-Base-36 --tiered=true --unstable=false --taskname=tiered-storage-cluster --machinearch=arm64 --instancetype=is4gen.4xlarge
     plugins:
-      - docker#v5.4.0:
+      - docker#v5.8.0:
           image: glrp/atgt:latest
           environment:
             - DA_AWS_ACCESS_KEY_ID
@@ -86,7 +102,7 @@ steps:
     key: tiered-up-unstable-fedora-is4
     command: .buildkite/scripts/standup-cluster.sh --prefix=rp-un-tier-fd-is --distro=Fedora-Cloud-Base-36 --tiered=true --unstable=true --taskname=tiered-storage-cluster --machinearch=arm64 --instancetype=is4gen.4xlarge
     plugins:
-      - docker#v5.4.0:
+      - docker#v5.8.0:
           image: glrp/atgt:latest
           environment:
             - DA_AWS_ACCESS_KEY_ID

diff --git a/.buildkite/scripts/standup-gcp-cluster.sh b/.buildkite/scripts/standup-gcp-cluster.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Parse command line arguments
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --tf_dir) TF_DIR="$2"; shift ;;
+        --prefix) PREFIX="$2"; shift ;;
+        --gcp_creds) GCP_CREDS="$2"; shift ;;
+        --cluster_type) TASK_NAME="$2"; shift ;;
+        --image) IMAGE="$2"; shift ;;
+        *) echo "Unknown parameter passed: $1"; exit 1 ;;
+    esac
+    shift
+done
+
+# Check if TF_DIR and PREFIX are set
+if [ -z "$TF_DIR" ] || [ -z "$PREFIX" ] || [ -z "$GCP_CREDS" ] || [ -z "$TASK_NAME" ]; then
+    echo "TF_DIR : $TF_DIR"
+    echo "TASK_NAME : $TASK_NAME"
+    echo "TF_DIR, PREFIX, CLUSTER_TYPE and GCP_CREDS must be set. Exiting."
+    exit 1
+fi
+
+cd "$TF_DIR" || exit 1
+export HOSTS_FILE_DIR="$(pwd)/../../artifacts/hosts_gcp_${PREFIX}.ini"
+export KEY_FILE="$(pwd)/../../artifacts/testkey"
+if [ "$TF_DIR" == "gcp" ]; then
+  export HOSTS_FILE_DIR="$(pwd)/../artifacts/hosts_gcp_${PREFIX}.ini"
+  export KEY_FILE="$(pwd)/../artifacts/testkey"
+fi
+
+ssh-keygen -t rsa -b 4096 -C "[email protected]" -N "" -f "$KEY_FILE" <<< y && chmod 0700 "$KEY_FILE"
+
+# Trap to handle terraform destroy on exit
+trap cleanup EXIT INT TERM
+cleanup() {
+    error_code=$?
+    terraform destroy --auto-approve --var="gcp_creds=$GCP_CREDS" --var="deployment_prefix=$PREFIX" --var="public_key_path=$KEY_FILE" --var="project_name=t" --var="hosts_file=$HOSTS_FILE_DIR"
+    rm -rf /app/ansible/tls
+    rm -f "$KEY_FILE"
+    rm -f "${KEY_FILE}.pub"
+    exit $error_code
+}
+
+terraform init
+terraform apply --auto-approve  --var="image=$IMAGE" --var="deployment_prefix=$PREFIX" --var="gcp_creds=$GCP_CREDS" --var="public_key_path=$KEY_FILE.pub" --var="project_name=hallowed-ray-376320" --var="hosts_file=$HOSTS_FILE_DIR"
+
+echo "building cluster"
+DEPLOYMENT_ID=$PREFIX DISTRO=$DISTRO IS_USING_UNSTABLE=$UNSTABLE CLOUD_STORAGE_CREDENTIALS_SOURCE="gcp_instance_metadata" SQUID_ACL_LOCALNET="10.0.0.0/24" CLOUD_PROVIDER="gcp" task "create-$TASK_NAME"
+error_code=$?
+if [ $error_code -ne 0 ]; then
+  echo "error in ansible standup $TASK_NAME"
+  exit 1
+fi
+
+echo "testing cluster"
+DEPLOYMENT_ID=$PREFIX DISTRO=$DISTRO CLOUD_PROVIDER="gcp" task "test-$TASK_NAME"
+error_code=$?
+if [ $error_code -ne 0 ]; then
+  echo "error in test $TASK_NAME"
+  exit 1
+fi
+
+# Trap will handle destroy so just exit
+exit $?
diff --git a/.buildkite/scripts/test-proxy-cluster.sh b/.buildkite/scripts/test-proxy-cluster.sh
@@ -15,6 +15,9 @@ while [ $# -gt 0 ]; do
     --sshkey=*)
       SSHKEY="${1#*=}"
       ;;
+    --cloud=*)
+      CLOUD_PROVIDER="${1#*=}"
+      ;;
     *)
       echo "Invalid argument: $1"
       exit 1
@@ -68,13 +71,23 @@ echo "consuming from topic"
 testoutput=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -i $SSHKEY $CLIENT_SSH_USER@$CLIENT_PUBLIC_IP 'rpk topic consume testtopic --brokers '"$REDPANDA_BROKERS"' --tls-truststore '"$PATH_TO_CA_CRT"' -v -o :end')
 echo $testoutput | grep squirrels || exit 1
 
-echo "checking that bucket is not empty"
-# Check if the bucket is empty
-object_count=$(aws s3api list-objects --bucket "${BUCKET_NAME}" --region us-west-2 --output json | jq '.Contents | length')
-
-if [ "$object_count" -gt 0 ]; then
+if [ "$CLOUD_PROVIDER" == "gcp" ]; then
+  echo "checking that gcp bucket is not empty"
+  echo "$DEVEX_GCP_CREDS_BASE64" | base64 -d > /tmp/gcp_creds.json
+  export GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcp_creds.json"
+  export CLOUDSDK_CORE_PROJECT=hallowed-ray-376320
+  gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS
+  echo $BUCKET_NAME
+  if [ $(gcloud storage ls $(gcloud storage ls | grep ${BUCKET_NAME%-bucket}) | wc -l) -gt 1 ]; then
     echo "success"
     exit 0
+  fi
+else
+  echo "checking that aws bucket is not empty"
+  # Check if the bucket is empty
+  object_count=$(aws s3api list-objects --bucket "${BUCKET_NAME}" --region us-west-2 --output json | jq '.Contents | length')
+  echo "success"
+  exit 0
 fi
 
 echo "fail"

diff --git a/.buildkite/scripts/test-tiered-storage-cluster.sh b/.buildkite/scripts/test-tiered-storage-cluster.sh
@@ -15,6 +15,9 @@ while [ $# -gt 0 ]; do
     --bucket=*)
       BUCKET_NAME="${1#*=}"
       ;;
+    --cloud=*)
+      CLOUD_PROVIDER="${1#*=}"
+      ;;
     *)
       echo "Invalid argument: $1"
       exit 1
@@ -71,15 +74,25 @@ testoutput=$("${PATH_TO_RPK_FILE}" topic consume testtopic --brokers "$REDPANDA_
 echo $testoutput | grep squirrels || exit 1
 
 echo "testing schema registry"
-for ip_port in $(echo $REDPANDA_REGISTRY | tr ',' ' '); do curl $ip_port/subjects -k --cacert "$PATH_TO_CA_CRT" ; done 
-
-echo "checking that bucket is not empty"
-# Check if the bucket is empty
-object_count=$(aws s3api list-objects --bucket "${BUCKET_NAME}" --region us-west-2 --output json | jq '.Contents | length')
+for ip_port in $(echo $REDPANDA_REGISTRY | tr ',' ' '); do curl $ip_port/subjects -k --cacert "$PATH_TO_CA_CRT" ; done
 
-if [ "$object_count" -gt 0 ]; then
+if [ "$CLOUD_PROVIDER" == "gcp" ]; then
+  echo "checking that gcp bucket is not empty"
+  echo "$DEVEX_GCP_CREDS_BASE64" | base64 -d > /tmp/gcp_creds.json
+  export GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcp_creds.json"
+  export CLOUDSDK_CORE_PROJECT=hallowed-ray-376320
+  gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS
+  echo $BUCKET_NAME
+  if [ $(gcloud storage ls $(gcloud storage ls | grep ${BUCKET_NAME%-bucket}) | wc -l) -gt 1 ]; then
     echo "success"
     exit 0
+  fi
+else
+  echo "checking that aws bucket is not empty"
+  # Check if the bucket is empty
+  object_count=$(aws s3api list-objects --bucket "${BUCKET_NAME}" --region us-west-2 --output json | jq '.Contents | length')
+  echo "success"
+  exit 0
 fi
 
 echo "fail"

diff --git a/Dockerfile_FEDORA b/Dockerfile_FEDORA
@@ -3,10 +3,12 @@ FROM fedora:36
 ENV DA_AWS_ACCESS_KEY_ID="default"
 ENV DA_AWS_SECRET_ACCESS_KEY="default"
 ENV AWS_DEFAULT_REGION="default"
+ENV GCP_CREDS="default"
 
 # Install required packages
 RUN dnf -y update \
     && dnf install -y unzip wget glibc curl vim git ansible jq openssl \
+    && dnf clean all \
     && rm -rf /var/cache/dnf/*
 
 # Install Terraform
@@ -19,6 +21,13 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv
 && unzip awscliv2.zip \
 && ./aws/install
 
+RUN curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-448.0.0-linux-x86_64.tar.gz && \
+    tar -zxvf google-cloud-cli-448.0.0-linux-x86_64.tar.gz && \
+    ./google-cloud-sdk/install.sh --quiet && \
+    rm google-cloud-cli-448.0.0-linux-x86_64.tar.gz
+
+ENV PATH="/google-cloud-sdk/bin:${PATH}"
+
 # Install task
 RUN curl -sSLf "https://github.com/go-task/task/releases/download/v3.21.0/task_linux_amd64.tar.gz" | tar -xz -C /usr/local/bin
 

diff --git a/Dockerfile_UBUNTU b/Dockerfile_UBUNTU
@@ -4,21 +4,20 @@ ARG DEBIAN_FRONTEND=noninteractive
 ENV DA_AWS_ACCESS_KEY_ID="default"
 ENV DA_AWS_SECRET_ACCESS_KEY="default"
 ENV AWS_DEFAULT_REGION="default"
+ENV GCP_CREDS="default"
 
 # Install required packages
 RUN apt-get update \
-    && apt install unzip -y \
-    && apt install wget -y \
-    && apt install curl -y \
-    && apt install vim -y \
-    && apt-get install -y git \
-    && apt-get install -y software-properties-common \
+    && apt install -y unzip wget curl vim git software-properties-common awscli jq lsb-release gnupg \
     && apt-add-repository --yes --update ppa:ansible/ansible \
     && apt-get install -y ansible \
-    && apt-get install -y awscli \
-    && apt-get install -y jq \
+    && echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \
+    && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg  add - \
+    && apt-get update  \
+    && apt-get install -y google-cloud-sdk \
     && rm -rf /var/lib/apt/lists/*
 
+
 # Install Terraform
 RUN wget https://releases.hashicorp.com/terraform/1.4.5/terraform_1.4.5_linux_amd64.zip \
     && unzip terraform_1.4.5_linux_amd64.zip -d /usr/local/bin \

diff --git a/Taskfile.yaml b/Taskfile.yaml
@@ -118,6 +118,7 @@ tasks:
       SKIP_TAGS: '{{ default "" .SKIP_TAGS }}'
     cmds:
       - mkdir -p {{.ARTIFACT_DIR}}/logs
+      - echo ansible-playbook ansible/{{.ANSIBLE_PLAYBOOK}} --private-key {{.PRIVATE_KEY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} {{ .SKIP_TAGS }} {{.CLI_ARGS}}
       - ansible-playbook ansible/{{.ANSIBLE_PLAYBOOK}} --private-key {{.PRIVATE_KEY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} {{ .SKIP_TAGS }} {{.CLI_ARGS}}
 
   create-tls-cluster:
@@ -162,11 +163,12 @@ tasks:
       SEGMENT_UPLOAD_INTERVAL: '{{ default "1" .SEGMENT_UPLOAD_INTERVAL }}'
       IS_USING_UNSTABLE: '{{ default false .IS_USING_UNSTABLE }}'
       ANSIBLE_PLAYBOOK: '{{ default "provision-tiered-storage-cluster.yml" .ANSIBLE_PLAYBOOK }}'
+      CLOUD_STORAGE_CREDENTIALS_SOURCE: '{{ default "aws_instance_metadata" .CLOUD_STORAGE_CREDENTIALS_SOURCE}}'
     cmds:
       - task: basic
         vars: {
             ANSIBLE_PLAYBOOK: "{{ .ANSIBLE_PLAYBOOK }}",
-            CLI_ARGS: "--extra-vars redpanda='{\"cluster\":{\"cloud_storage_segment_max_upload_interval_sec\":\"{{.SEGMENT_UPLOAD_INTERVAL}}\"}}'",
+            CLI_ARGS: "--extra-vars cloud_storage_credentials_source='{{ .CLOUD_STORAGE_CREDENTIALS_SOURCE }}' --extra-vars redpanda='{\"cluster\":{\"cloud_storage_segment_max_upload_interval_sec\":\"{{.SEGMENT_UPLOAD_INTERVAL}}\"}}'",
             IS_USING_UNSTABLE: "{{ .IS_USING_UNSTABLE }}"
         }
 
@@ -187,13 +189,14 @@ tasks:
     vars:
       SEGMENT_UPLOAD_INTERVAL: '{{ default "1" .SEGMENT_UPLOAD_INTERVAL }}'
       IS_USING_UNSTABLE: '{{ default false .IS_USING_UNSTABLE }}'
-      ANSIBLE_PLAYBOOK: '{{ default "provision-private-proxied-cluster.yml" .ANSIBLE_PLAYBOOK }}'
+      ANSIBLE_PLAYBOOK: '{{ default "proxy/provision-private-proxied-cluster.yml" .ANSIBLE_PLAYBOOK }}'
       SKIP_TAGS: '{{ default "" .SKIP_TAGS }}'
+      SQUID_ACL_LOCALNET: '{{ default "" .SQUID_ACL_LOCALNET }}'
     cmds:
       - task: basic
         vars: {
             ANSIBLE_PLAYBOOK: "{{ .ANSIBLE_PLAYBOOK }}",
-            CLI_ARGS: "--extra-vars redpanda='{\"cluster\":{\"cloud_storage_segment_max_upload_interval_sec\":\"{{.SEGMENT_UPLOAD_INTERVAL}}\"}}'",
+            CLI_ARGS: "--extra-vars '{\"squid_acl_localnet\": [\"{{.SQUID_ACL_LOCALNET}}\"]}' --extra-vars redpanda='{\"cluster\":{\"cloud_storage_segment_max_upload_interval_sec\":\"{{.SEGMENT_UPLOAD_INTERVAL}}\"}}'",
             IS_USING_UNSTABLE: "{{ .IS_USING_UNSTABLE }}",
             SKIP_TAGS: "{{ .SKIP_TAGS }}"
         }
@@ -227,7 +230,7 @@ tasks:
       RPK_LOC: '{{.ARTIFACT_DIR}}/bin/rpk'
       # this needs to go somewhere else badly
       CA_CRT: '{{ default "ansible/tls/ca/ca.crt" .CA_CRT }}'
-      ANSIBLE_INVENTORY: '{{.ARTIFACT_DIR}}/hosts_{{.CLOUD_PROVIDER}}_{{.DEPLOYMENT_ID}}.ini'
+      ANSIBLE_INVENTORY: '{{.ANSIBLE_INVENTORY}}'
     cmds:
       - '{{.TASKFILE_DIR}}/.buildkite/scripts/test-tls-cluster.sh --hosts={{.ANSIBLE_INVENTORY}} --cert={{.CA_CRT}} --rpk={{.RPK_LOC}}'
 
@@ -239,14 +242,14 @@ tasks:
       RPK_LOC: '{{.ARTIFACT_DIR}}/bin/rpk'
       # this needs to go somewhere else badly
       CA_CRT: '{{ default "ansible/tls/ca/ca.crt" .CA_CRT }}'
-      ANSIBLE_INVENTORY: '{{.ARTIFACT_DIR}}/hosts_{{.CLOUD_PROVIDER}}_{{.DEPLOYMENT_ID}}.ini'
+      ANSIBLE_INVENTORY: '{{.ANSIBLE_INVENTORY}}'
     cmds:
       - '{{.TASKFILE_DIR}}/.buildkite/scripts/test-tiered-storage-cluster.sh --hosts={{.ANSIBLE_INVENTORY}} --cert={{.CA_CRT}} --rpk={{.RPK_LOC}} --bucket={{.BUCKET_NAME}}'
 
   test-proxy-cluster:
     desc: tests that a proxied cluster is WAD
     vars:
       CA_CRT: '{{ default "/opt/rpk/certs/ca.crt" .CA_CRT }}'
-      ANSIBLE_INVENTORY: '{{.ARTIFACT_DIR}}/hosts_{{.CLOUD_PROVIDER}}_{{.DEPLOYMENT_ID}}.ini'
+      ANSIBLE_INVENTORY: '{{.ANSIBLE_INVENTORY}}'
     cmds:
       - '{{.TASKFILE_DIR}}/.buildkite/scripts/test-proxy-cluster.sh --hosts={{.ANSIBLE_INVENTORY}} --cert={{.CA_CRT}} --bucket={{.BUCKET_NAME}} --sshkey=artifacts/testkey'
diff --git a/gcp/readme.md → gcp/README.md b/gcp/readme.md → gcp/README.md