Use cache from parent commits

DataDog · Nov 27, 2024 · 83d15d2 · 83d15d2
1 parent 8108b3e
commit 83d15d2
Showing 1 changed file with 121 additions and 15 deletions.
diff --git a/.github/workflows/build-ruby.yml b/.github/workflows/build-ruby.yml
@@ -223,6 +223,7 @@ jobs:
           echo "SRC=src/engines/${{ matrix.engine }}/${{ matrix.version }}" >> $GITHUB_OUTPUT
           echo "IMAGE=${{ env.REGISTRY }}/${{ env.REPO }}/engines/${{ matrix.engine }}" >> $GITHUB_OUTPUT
           echo "RELEASE_TAG=${{ matrix.version }}-${{ matrix.libc }}" >> $GITHUB_OUTPUT
+          echo "COMMIT_TAG=${{ matrix.version }}-${{ matrix.libc }}-g${{ github.sha }}" >> $GITHUB_OUTPUT
           echo "TAG=${{ matrix.version }}-${{ matrix.libc }}-gha${{ github.run_id }}-g${{ github.sha }}" >> $GITHUB_OUTPUT
           echo "DOCKERFILE=src/engines/${{ matrix.engine }}/${{ matrix.version }}/Dockerfile.${{ matrix.libc }}" >> $GITHUB_OUTPUT
           echo "DOCKER_PLATFORMS=$(echo ${{ join(matrix.arch) }} | tr ',' '\n' | sed 's/^/linux\//' | paste -s -d, -)" >> $GITHUB_OUTPUT
@@ -238,11 +239,23 @@ jobs:
       - name: Build single-arch image (x86_64)
         if: ${{ contains(matrix.arch, 'x86_64') }}
         run: |
-          docker buildx build ${{ steps.vars.outputs.SRC }} --cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }} --output=type=image,push=false --platform linux/x86_64 -f ${{ steps.vars.outputs.DOCKERFILE }}
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }})
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --output=type=image,push=false --platform linux/x86_64 -f ${{ steps.vars.outputs.DOCKERFILE }}
       - name: Tag single-arch image (x86_64)
         if: ${{ contains(matrix.arch, 'x86_64') }}
         run: |
-          docker buildx build ${{ steps.vars.outputs.SRC }} --cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }} --load --platform linux/x86_64 -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.TAG }}
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }})
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --load --platform linux/x86_64 -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.TAG }}
       - name: Test single-arch image (x86_64)
         if: ${{ contains(matrix.arch, 'x86_64') }}
         run: |
@@ -257,13 +270,23 @@ jobs:
       - name: Tag single-arch image (x86_64)
         if: ${{ contains(matrix.arch, 'x86_64') }}
         run: |
-          docker buildx build ${{ steps.vars.outputs.SRC }} --cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }} --load --platform linux/x86_64 -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}
-          docker image list
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }})
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --load --platform linux/x86_64 -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}
       - name: Build single-arch compiler image (x86_64)
         if: ${{ contains(matrix.arch, 'x86_64') && matrix.libc == 'musl' }}
         run: |
-          docker image list # image not in builder=container!
-          docker buildx build ${{ steps.vars.outputs.SRC }} --cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc --output=type=image,push=false --platform linux/x86_64 -f ${{ steps.vars.outputs.DOCKERFILE }}.gcc
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc)
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --output=type=image,push=false --platform linux/x86_64 -f ${{ steps.vars.outputs.DOCKERFILE }}.gcc
       # TODO: add CI tag and test
 
       # Then, build image for aarch64 which, being emulated under qemu, is slower
@@ -278,11 +301,23 @@ jobs:
       - name: Build single-arch image (aarch64)
         if: ${{ contains(matrix.arch, 'aarch64') }}
         run: |
-          docker buildx build ${{ steps.vars.outputs.SRC }} --cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }} --output=type=image,push=false --platform linux/aarch64 -f ${{ steps.vars.outputs.DOCKERFILE }}
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }})
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --output=type=image,push=false --platform linux/aarch64 -f ${{ steps.vars.outputs.DOCKERFILE }}
       - name: Tag single-arch image (aarch64)
         if: ${{ contains(matrix.arch, 'aarch64') }}
         run: |
-          docker buildx build ${{ steps.vars.outputs.SRC }} --cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }} --load --platform linux/aarch64 -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.TAG }}
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }})
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --load --platform linux/aarch64 -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.TAG }}
       - name: Test single-arch image (aarch64)
         if: ${{ contains(matrix.arch, 'aarch64') }}
         run: |
@@ -297,11 +332,23 @@ jobs:
       - name: Tag single-arch image (aarch64)
         if: ${{ contains(matrix.arch, 'aarch64') }}
         run: |
-          docker buildx build ${{ steps.vars.outputs.SRC }} --cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }} --load --platform linux/aarch64 -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }})
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --load --platform linux/aarch64 -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}
       - name: Build single-arch compiler image (aarch64)
         if: ${{ contains(matrix.arch, 'aarch64') && matrix.libc == 'musl' }}
         run: |
-          docker buildx build ${{ steps.vars.outputs.SRC }} --cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc --output=type=image,push=false --platform linux/aarch64 -f ${{ steps.vars.outputs.DOCKERFILE }}.gcc
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc)
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --output=type=image,push=false --platform linux/aarch64 -f ${{ steps.vars.outputs.DOCKERFILE }}.gcc
       # TODO: add CI tag and test
 
       # Finally, assemble multi-arch image for a combined push to the registry
@@ -312,26 +359,85 @@ jobs:
           echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
       - name: Push CI run image (${{ join(matrix.arch, ', ') }})
         run: |
-          docker buildx build ${{ steps.vars.outputs.SRC }} --cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }} --output=type=image,push=true --build-arg BUILDKIT_INLINE_CACHE=1 --platform ${{ steps.vars.outputs.DOCKER_PLATFORMS }} -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.TAG }}
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }})
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --output=type=image,push=true --build-arg BUILDKIT_INLINE_CACHE=1 --platform ${{ steps.vars.outputs.DOCKER_PLATFORMS }} -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.TAG }}
+      - name: Push commit image (${{ join(matrix.arch, ', ') }})
+        run: |
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }})
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --output=type=image,push=true --build-arg BUILDKIT_INLINE_CACHE=1 --platform ${{ steps.vars.outputs.DOCKER_PLATFORMS }} -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-g${{ github.sha }}
       - name: Push release image (${{ join(matrix.arch, ', ') }})
         if: ${{ inputs.push }}
         run: |
-          docker buildx build ${{ steps.vars.outputs.SRC }} --cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }} --output=type=image,push=true --build-arg BUILDKIT_INLINE_CACHE=1 --platform ${{ steps.vars.outputs.DOCKER_PLATFORMS }} -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }})
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --output=type=image,push=true --build-arg BUILDKIT_INLINE_CACHE=1 --platform ${{ steps.vars.outputs.DOCKER_PLATFORMS }} -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}
 
       # TODO: hardcoded, reuse strip-tags directive instead (or better, unify gnu)
       - name: Push unqualified release image (${{ join(matrix.arch, ', ') }})
         if: ${{ inputs.push && matrix.libc == 'gnu' }}
         run: |
-          docker buildx build ${{ steps.vars.outputs.SRC }} --cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }} --output=type=image,push=true --build-arg BUILDKIT_INLINE_CACHE=1 --platform ${{ steps.vars.outputs.DOCKER_PLATFORMS }} -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ matrix.version }}
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }})
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --output=type=image,push=true --build-arg BUILDKIT_INLINE_CACHE=1 --platform ${{ steps.vars.outputs.DOCKER_PLATFORMS }} -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ matrix.version }}
 
       # TODO: hardcoded, reuse append-tags directive instead (or better, unify gnu+centos)
+      - name: Push commit compiler image (${{ join(matrix.arch, ', ') }})
+        if: ${{ inputs.push && (matrix.libc == 'gnu' || matrix.libc == 'centos') }}
+        run: |
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc)
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --output=type=image,push=true --build-arg BUILDKIT_INLINE_CACHE=1 --platform ${{ steps.vars.outputs.DOCKER_PLATFORMS }} -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc-g${sha}
       - name: Push release compiler image (${{ join(matrix.arch, ', ') }})
         if: ${{ inputs.push && (matrix.libc == 'gnu' || matrix.libc == 'centos') }}
         run: |
-          docker buildx build ${{ steps.vars.outputs.SRC }} --cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc --output=type=image,push=true --build-arg BUILDKIT_INLINE_CACHE=1 --platform ${{ steps.vars.outputs.DOCKER_PLATFORMS }} -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc)
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --output=type=image,push=true --build-arg BUILDKIT_INLINE_CACHE=1 --platform ${{ steps.vars.outputs.DOCKER_PLATFORMS }} -f ${{ steps.vars.outputs.DOCKERFILE }} --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc
 
       # TODO: hardcoded musl, unify gnu instead
+      - name: Push commit compiler image (${{ join(matrix.arch, ', ') }})
+        if: ${{ inputs.push && matrix.libc == 'musl' }}
+        run: |
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc)
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --output=type=image,push=true --build-arg BUILDKIT_INLINE_CACHE=1 --platform ${{ steps.vars.outputs.DOCKER_PLATFORMS }} -f ${{ steps.vars.outputs.DOCKERFILE }}.gcc --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc-g${sha}
       - name: Push release compiler image (${{ join(matrix.arch, ', ') }})
         if: ${{ inputs.push && matrix.libc == 'musl' }}
         run: |
-          docker buildx build ${{ steps.vars.outputs.SRC }} --cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc --output=type=image,push=true --build-arg BUILDKIT_INLINE_CACHE=1 --platform ${{ steps.vars.outputs.DOCKER_PLATFORMS }} -f ${{ steps.vars.outputs.DOCKERFILE }}.gcc --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc
+          cache_from=()
+          parents="$(git rev-list --parents -n 1 ${{ github.sha }})"
+          for sha in $parents; do
+            cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc-g${sha})
+          done
+          cache_from+=(--cache-from=type=registry,ref=${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc)
+          docker buildx build ${{ steps.vars.outputs.SRC }} "${cache_from[@]}" --output=type=image,push=true --build-arg BUILDKIT_INLINE_CACHE=1 --platform ${{ steps.vars.outputs.DOCKER_PLATFORMS }} -f ${{ steps.vars.outputs.DOCKERFILE }}.gcc --tag ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.RELEASE_TAG }}-gcc