MDBF-770 - GitHub Reusable Workflows

The core idea is to use GitHub reusable workflows to divide image builds into groups of parent workflows. Based on the "matrix" in bbw_build_container.yml, parent workflows were created to reflect Dockerfile usage. Each workflow monitors changes in the corresponding Dockerfile, allowing for more granular control over which images are built. A master workflow, "build-workflow-dispatcher," is available to trigger all the child workflows manually or when the template is changed. Rollout Plan: - In bbw_build_container_template.yml, the PUSH to QUAY/GHCR is commented out so we can push this commit and perform regression tests against the current bbw_build_container workflow. - All pending contributions to bbw_build_container should be rebased and adapted to this new workflow by either changing the template file or the corresponding parent workflow. - Our forks should be synced after this is merged. - when we are ready to activate PUSH to registries, we uncomment QUAY/GHCR blocks of code in the template file and disable the old workflow - the old workflow should stay disable for a while in case of a rollback
RazvanLiviuVarzaru · Aug 8, 2024 · 0dda099 · 0dda099
1 parent 2c54f6e
commit 0dda099
Show file tree

Hide file tree

Showing 12 changed files with 632 additions and 0 deletions.
diff --git a/.github/workflows/bbw_build_container_template.yml b/.github/workflows/bbw_build_container_template.yml
@@ -0,0 +1,171 @@
+---
+name: bbw-build-container-template
+
+on:
+  workflow_call:
+    inputs:
+      dockerfile:
+        required: true
+        type: string
+      image:
+        required: true
+        type: string
+      platforms:
+        required: true
+        type: string
+      tag:
+        required: false
+        type: string
+      runner:
+        required: false
+        type: string
+      clang_version:
+        required: false
+        type: string
+      branch:
+        required: false
+        type: string
+      install_valgrind:
+        required: false
+        type: string
+
+jobs:
+  build:
+    runs-on: ${{ inputs.runner || 'ubuntu-22.04' }}
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
+    name: ${{ inputs.image }} (${{ inputs.tag }} ${{ inputs.platforms }})
+    env:
+      BUILD_RHEL: false
+      DEPLOY_IMAGES: false
+      WORKDIR: ci_build_images
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up env vars
+        run: |
+          set -vx
+          [[ -n "${{ inputs.image }}" ]] || {
+            echo "Missing base image (FROM)"
+            exit 1
+          }
+          if [[ -n "${{ inputs.tag }}" ]]; then
+            echo "IMG=${{ inputs.tag }}" >>$GITHUB_ENV
+          else
+            TAG_TMP=${{ inputs.image }}
+            echo "IMG=${TAG_TMP/:/}" >>$GITHUB_ENV
+          fi
+          echo "REPO=bb-worker" >>$GITHUB_ENV
+      - name: Generate Dockerfile and necessary files
+        run: |
+          cd ${{ env.WORKDIR }}
+          cat ${{ inputs.dockerfile }} qpress.Dockerfile buildbot-worker.Dockerfile >$GITHUB_WORKSPACE/Dockerfile
+          cp -r qpress $GITHUB_WORKSPACE
+      - name: opensuse extra
+        if: contains(inputs.tag, 'opensuse') || contains(inputs.tag, 'sles')
+        run: |
+          cp ${{ env.WORKDIR }}/mariadb_zypper_expect $GITHUB_WORKSPACE
+      - name: No wsrep on 32 bit platforms
+        if: >
+          (contains(inputs.platforms, 'linux/386'))
+        run: |
+          sed -i -e '/WSREP_PROVIDER/d' $GITHUB_WORKSPACE/Dockerfile
+      - name: Check Dockerfile with hadolint
+        run: |
+          docker run -i -v $(pwd):/mnt -w /mnt ghcr.io/hadolint/hadolint:latest hadolint /mnt/Dockerfile
+      - name: Install qemu-user-static
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y qemu-user-static
+      - name: Build image
+        run: |
+          podman manifest create ${{ env.REPO }}:${{ env.IMG }}
+          for arch in $(echo ${{ inputs.platforms }} | sed 's/,/ /g'); do
+            msg="Build $arch:"
+            line="${msg//?/=}"
+            printf "\n${line}\n${msg}\n${line}\n"
+            podman buildx build --tag ${{ env.REPO }}:${{ env.IMG }}-${arch//\//-} \
+              --platform $arch \
+              --manifest ${{ env.REPO }}:${{ env.IMG }} \
+              -f $GITHUB_WORKSPACE/Dockerfile \
+              --build-arg BASE_IMAGE=${{ inputs.image }} \
+              --build-arg CLANG_VERSION=${{ inputs.clang_version }} \
+              --build-arg MARIADB_BRANCH=${{ inputs.branch }} \
+              --build-arg INSTALL_VALGRIND="${{ inputs.install_valgrind }}"
+          done
+          podman images
+      - name: Push images to local registry
+        run: |
+          podman manifest push --tls-verify=0 \
+            --all ${{ env.REPO }}:${{ env.IMG }} \
+            docker://localhost:5000/${{ env.REPO }}:${{ env.IMG }}
+      - name: Check multi-arch container
+        run: |
+          # make some space on the runner
+          if [[ -d $HOME/.local/share/containers ]]; then
+            sudo rm -rf $HOME/.local/share/containers
+          fi
+          for p in ${{ inputs.platforms }}; do
+            platform="${p/,/}"
+            image="localhost:5000/bb-worker:${{ env.IMG }}"
+            msg="Testing docker image $image on platform $platform"
+            line="${msg//?/=}"
+            printf "\n${line}\n${msg}\n${line}\n"
+            docker pull -q --platform "$platform" "$image"
+            docker run -i "$image" buildbot-worker --version
+            docker run -i "$image" dumb-init twistd --pidfile= -y /home/buildbot/buildbot.tac
+            docker run -u root -i "$image" bash -c "touch /tmp/foo && qpress -r /tmp /root/qpress.qp"
+          done
+      - name: Check for registry credentials
+        run: |
+          missing=()
+          [[ -n "${{ secrets.QUAY_USER }}" ]] || missing+=(QUAY_USER)
+          [[ -n "${{ secrets.QUAY_TOKEN }}" ]] || missing+=(QUAY_TOKEN)
+          for i in "${missing[@]}"; do
+            echo "Missing github secret: $i"
+          done
+          if (( ${#missing[@]} == 0 )); then
+            echo "DEPLOY_IMAGES=true" >> $GITHUB_ENV
+          else
+            echo "Not pushing images to registry"
+          fi
+      - name: Login to ghcr.io
+        if: ${{ env.DEPLOY_IMAGES == 'true' }}
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Uncomment this when we disable the old workflow
+      # - name: Push images to ghcr.io
+      #   if: ${{ env.DEPLOY_IMAGES == 'true' }}
+      #   run: |
+      #     msg="Push docker image to ghcr.io (${{ env.IMG }})"
+      #     line="${msg//?/=}"
+      #     printf "\n${line}\n${msg}\n${line}\n"
+      #     skopeo copy --all --src-tls-verify=0 \
+      #       docker://localhost:5000/${{ env.REPO }}:${{ env.IMG }} \
+      #       docker://ghcr.io/${GITHUB_REPOSITORY,,}/${{ env.REPO }}:${{ env.IMG }}
+
+      - name: Login to registry
+        if: ${{ env.DEPLOY_IMAGES == 'true' }}
+        uses: docker/login-action@v2
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USER }}
+          password: ${{ secrets.QUAY_TOKEN }}
+
+      # Uncomment this when we disable the old workflow
+      # - name: Push images to quay.io
+      #   if: ${{ env.DEPLOY_IMAGES == 'true' }}
+      #   run: |
+      #     msg="Push docker image to quay.io (${{ env.IMG }})"
+      #     line="${msg//?/=}"
+      #     printf "\n${line}\n${msg}\n${line}\n"
+      #     skopeo copy --all --src-tls-verify=0 \
+      #     docker://localhost:5000/${{ env.REPO }}:${{ env.IMG }} \
+      #     docker://quay.io/mariadb-foundation/${{ env.REPO }}:${{ env.IMG }}
diff --git a/.github/workflows/build-centos-based.yml b/.github/workflows/build-centos-based.yml
@@ -0,0 +1,33 @@
+name: Build CentOS based images
+
+on:
+  push:
+    paths:
+      - 'ci_build_images/centos.Dockerfile'
+      - 'ci_build_images/qpress.Dockerfile'
+      - 'ci_build_images/buildbot-worker.Dockerfile'
+      - .github/workflows/build-centos-based.yml
+  pull_request:
+    paths:
+      - 'ci_build_images/centos.Dockerfile'
+      - 'ci_build_images/qpress.Dockerfile'
+      - 'ci_build_images/buildbot-worker.Dockerfile'
+      - .github/workflows/build-centos-based.yml
+  workflow_call:
+
+jobs:
+  build-images:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: almalinux:8
+            platforms: linux/amd64, linux/arm64/v8
+          - image: rockylinux:8
+            platforms: linux/amd64, linux/arm64/v8
+    uses: ./.github/workflows/bbw_build_container_template.yml
+    with:
+      dockerfile: centos.Dockerfile
+      image: ${{ matrix.image }}
+      platforms: ${{ matrix.platforms }}
+    secrets: inherit
diff --git a/.github/workflows/build-centos.pip-based.yml b/.github/workflows/build-centos.pip-based.yml
@@ -0,0 +1,45 @@
+name: Build CentOS:pip based images
+
+on:
+  push:
+    paths:
+      - 'ci_build_images/centos.Dockerfile'
+      - 'ci_build_images/pip.Dockerfile'
+      - 'ci_build_images/qpress.Dockerfile'
+      - 'ci_build_images/buildbot-worker.Dockerfile'
+      - .github/workflows/build-centos.pip-based.yml
+  pull_request:
+    paths:
+      - 'ci_build_images/centos.Dockerfile'
+      - 'ci_build_images/pip.Dockerfile'
+      - 'ci_build_images/qpress.Dockerfile'
+      - 'ci_build_images/buildbot-worker.Dockerfile'
+      - .github/workflows/build-centos.pip-based.yml
+
+  workflow_call:
+
+jobs:
+  build-images:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: almalinux:9
+            platforms: linux/amd64, linux/arm64/v8
+
+          - image: rockylinux:9
+            platforms: linux/amd64, linux/arm64/v8
+
+          - image: quay.io/centos/centos:stream9
+            platforms: linux/amd64, linux/arm64/v8, linux/ppc64le
+            tag: centosstream9
+            runner: ubuntu-24.04
+
+    uses: ./.github/workflows/bbw_build_container_template.yml
+    with:
+      dockerfile: centos.Dockerfile pip.Dockerfile
+      image: ${{ matrix.image }}
+      platforms: ${{ matrix.platforms }}
+      runner: ${{ matrix.runner }}
+      tag: ${{ matrix.tag }}
+    secrets: inherit
diff --git a/.github/workflows/build-centos7.pip-based.yml b/.github/workflows/build-centos7.pip-based.yml
@@ -0,0 +1,33 @@
+name: Build CentOS7:pip based images
+
+on:
+  push:
+    paths:
+      - 'ci_build_images/centos7.Dockerfile'
+      - 'ci_build_images/pip.Dockerfile'
+      - 'ci_build_images/qpress.Dockerfile'
+      - 'ci_build_images/buildbot-worker.Dockerfile'
+      - .github/workflows/build-centos7.pip-based.yml
+  pull_request:
+    paths:
+      - 'ci_build_images/centos7.Dockerfile'
+      - 'ci_build_images/pip.Dockerfile'
+      - 'ci_build_images/qpress.Dockerfile'
+      - 'ci_build_images/buildbot-worker.Dockerfile'
+      - .github/workflows/build-centos7.pip-based.yml
+  workflow_call:
+
+jobs:
+  build-images:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: centos:7
+            platforms: linux/amd64
+    uses: ./.github/workflows/bbw_build_container_template.yml
+    with:
+      dockerfile: centos7.Dockerfile pip.Dockerfile
+      image: ${{ matrix.image }}
+      platforms: ${{ matrix.platforms }}
+    secrets: inherit
diff --git a/.github/workflows/build-debian-based.yml b/.github/workflows/build-debian-based.yml
@@ -0,0 +1,71 @@
+name: Build Debian based images
+
+on:
+  push:
+    paths:
+      - 'ci_build_images/debian.Dockerfile'
+      - 'ci_build_images/qpress.Dockerfile'
+      - 'ci_build_images/buildbot-worker.Dockerfile'
+      - .github/workflows/build-debian-based.yml
+  pull_request:
+    paths:
+      - 'ci_build_images/debian.Dockerfile'
+      - 'ci_build_images/qpress.Dockerfile'
+      - 'ci_build_images/buildbot-worker.Dockerfile'
+      - .github/workflows/build-debian-based.yml
+
+  workflow_call:
+
+jobs:
+  build-images:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: debian:11
+            platforms: linux/amd64, linux/arm64/v8, linux/ppc64le
+            branch: 10.11
+
+          - image: debian:12
+            platforms: linux/amd64, linux/arm64/v8, linux/ppc64le
+            branch: 10.11
+            tag: debian12
+
+          - image: debian:12
+            platforms: linux/386
+            branch: 10.11
+            tag: debian12-386
+
+          - image: debian:sid
+            platforms: linux/amd64, linux/arm64/v8, linux/ppc64le
+            branch: 10.11
+
+          - image: debian:sid
+            platforms: linux/386
+            branch: 10.11
+            tag: debiansid-386
+
+          - image: ubuntu:20.04
+            platforms: linux/amd64, linux/arm64/v8, linux/ppc64le, linux/s390x
+            branch: 10.11
+
+          - image: ubuntu:22.04
+            platforms: linux/amd64, linux/arm64/v8, linux/ppc64le, linux/s390x
+            branch: 10.11
+
+          - image: ubuntu:23.10
+            platforms: linux/amd64, linux/arm64/v8
+            branch: 10.11
+
+          - image: ubuntu:24.04
+            platforms: linux/amd64, linux/arm64/v8, linux/ppc64le, linux/s390x
+            branch: 10.11
+
+    uses: ./.github/workflows/bbw_build_container_template.yml
+    with:
+      dockerfile: debian.Dockerfile
+      image: ${{ matrix.image }}
+      platforms: ${{ matrix.platforms }}
+      tag: ${{ matrix.tag }}
+      branch: ${{ matrix.branch }}
+    secrets: inherit
diff --git a/.github/workflows/build-debian.aocc-based.yml b/.github/workflows/build-debian.aocc-based.yml
@@ -0,0 +1,37 @@
+name: Build Debian:aocc based images
+
+on:
+  push:
+    paths:
+      - 'ci_build_images/debian.Dockerfile'
+      - 'ci_build_images/aocc.Dockerfile'
+      - 'ci_build_images/qpress.Dockerfile'
+      - 'ci_build_images/buildbot-worker.Dockerfile'
+      - .github/workflows/build-debian.aocc-based.yml
+  pull_request:
+    paths:
+      - 'ci_build_images/debian.Dockerfile'
+      - 'ci_build_images/aocc.Dockerfile'
+      - 'ci_build_images/qpress.Dockerfile'
+      - 'ci_build_images/buildbot-worker.Dockerfile'
+      - .github/workflows/build-debian.aocc-based.yml
+  workflow_call:
+
+jobs:
+  build-images:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: debian:11
+            platforms: linux/amd64
+            branch: 10.11
+            tag: debian11-aocc
+    uses: ./.github/workflows/bbw_build_container_template.yml
+    with:
+      dockerfile: debian.Dockerfile aocc.Dockerfile
+      image: ${{ matrix.image }}
+      platforms: ${{ matrix.platforms }}
+      tag: ${{ matrix.tag }}
+      branch: ${{ matrix.branch }}
+    secrets: inherit