Only save caches on master (and not schedule event) (#261)

G-Research · Nov 4, 2024 · 2f32f8c · 2f32f8c
1 parent a91393a
commit 2f32f8c
Show file tree

Hide file tree

Showing 5 changed files with 112 additions and 36 deletions.
diff --git a/.github/actions/build-whl/action.yml b/.github/actions/build-whl/action.yml
@@ -34,20 +34,25 @@ runs:
       name: Binaries-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
       path: .
 
-  - name: Cache Maven packages
-    if: github.event_name != 'merge_group'
-    uses: actions/cache@v4
+  - name: Restore Maven packages cache
+    if: github.event_name != 'schedule'
+    uses: actions/cache/restore@v4
     with:
       path: ~/.m2/repository
       key: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
-      restore-keys: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
+      restore-keys:
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
 
-  - name: Cache Pip packages
-    if: github.event_name != 'merge_group'
-    uses: actions/cache@v4
+  - name: Restore Pip packages cache
+    if: github.event_name != 'schedule'
+    uses: actions/cache/restore@v4
     with:
       path: ~/.cache/pip
-      key: ${{ runner.os }}-pip-whl-${{ inputs.python-version }}-${{ inputs.spark-version }}
+      key: ${{ runner.os }}-pip-whl-${{ inputs.python-version }}-${{ hashFiles(format('python/requirements-{0}_{1}.txt', inputs.spark-compat-version, inputs.scala-compat-version)) }}
+      restore-keys:
+        ${{ runner.os }}-pip-whl-${{ inputs.python-version }}-${{ hashFiles(format('python/requirements-{0}_{1}.txt', inputs.spark-compat-version, inputs.scala-compat-version)) }}
+        ${{ runner.os }}-pip-whl-${{ inputs.python-version }}-
 
   - name: Setup Python
     uses: actions/setup-python@v5
@@ -72,6 +77,13 @@ runs:
       python test-release.py
     shell: bash
 
+  - name: Save Pip packages cache
+    if: github.event_name != 'schedule' && github.ref == 'refs/heads/master'
+    uses: actions/cache/save@v4
+    with:
+      path: ~/.cache/pip
+      key: ${{ runner.os }}-pip-whl-${{ inputs.python-version }}-${{ hashFiles(format('python/requirements-{0}_{1}.txt', inputs.spark-compat-version, inputs.scala-compat-version)) }}-${{ github.run_id }}}
+
   - name: Upload whl
     uses: actions/upload-artifact@v4
     with:

diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml
@@ -28,13 +28,15 @@ runs:
       git diff
     shell: bash
 
-  - name: Cache Maven packages
-    if: github.event_name != 'merge_group'
-    uses: actions/cache@v4
+  - name: Restore Maven packages cache
+    if: github.event_name != 'schedule'
+    uses: actions/cache/restore@v4
     with:
       path: ~/.m2/repository
       key: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
-      restore-keys: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
+      restore-keys:
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
 
   - name: Setup JDK ${{ inputs.java-compat-version }}
     uses: actions/setup-java@v4
@@ -51,6 +53,42 @@ runs:
       mvn --batch-mode install -Dspotless.check.skip -DskipTests -Dmaven.test.skip=true -Dgpg.skip
     shell: bash
 
+  - name: Save Maven packages cache
+    if: github.event_name != 'schedule' && github.ref == 'refs/heads/master'
+    uses: actions/cache/save@v4
+    with:
+      path: ~/.m2/repository
+      key: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}-${{ github.run_id }}}
+
+  # merely populate the cache for test-jvm and test-python actions
+  - name: Restore Spark Binaries cache
+    if: github.event_name != 'schedule' && github.ref == 'refs/heads/master' && ! contains(inputs.spark-version, '-SNAPSHOT')
+    uses: actions/cache/restore@v4
+    with:
+      path: ~/spark
+      key: ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}
+      restore-keys:
+        ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}
+
+  - name: Setup Spark Binaries
+    if: github.event_name != 'schedule' && github.ref == 'refs/heads/master' && ! contains(inputs.spark-version, '-SNAPSHOT')
+    env:
+      SPARK_PACKAGE: spark-${{ inputs.spark-version }}/spark-${{ inputs.spark-version }}-bin-hadoop${{ inputs.hadoop-version }}${{ inputs.scala-compat-version == '2.13' && '-scala2.13' || '' }}.tgz
+    run: |
+      if [[ ! -e ~/spark ]]
+      then
+        wget --progress=dot:giga "https://www.apache.org/dyn/closer.lua/spark/${SPARK_PACKAGE}?action=download" -O - | tar -xzC "${{ runner.temp }}"
+        archive=$(basename "${SPARK_PACKAGE}") bash -c "mv -v "${{ runner.temp }}/\${archive/%.tgz/}" ~/spark"
+      fi
+    shell: bash
+
+  - name: Save Spark Binaries cache
+    if: github.event_name != 'schedule' && github.ref == 'refs/heads/master' && ! contains(inputs.spark-version, '-SNAPSHOT')
+    uses: actions/cache/save@v4
+    with:
+      path: ~/spark
+      key: ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}-${{ github.run_id }}}
+
   - name: Upload Binaries
     uses: actions/upload-artifact@v4
     with:

diff --git a/.github/actions/check-compat/action.yml b/.github/actions/check-compat/action.yml
@@ -34,15 +34,15 @@ runs:
       name: Binaries-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
       path: .
 
-  - name: Cache Maven packages
-    if: github.event_name != 'merge_group'
-    uses: actions/cache@v4
+  - name: Restore Maven packages cache
+    if: github.event_name != 'schedule'
+    uses: actions/cache/restore@v4
     with:
       path: ~/.m2/repository
       key: ${{ runner.os }}-mvn-check-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
-      restore-keys: |
-        ${{ runner.os }}-mvn-check-${{ inputs.spark-version }}-${{ inputs.scala-version }}
-        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
+      restore-keys:
+        ${{ runner.os }}-mvn-check-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
+        ${{ runner.os }}-mvn-check-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
 
   - name: Setup JDK 1.8
     uses: actions/setup-java@v4
@@ -77,6 +77,13 @@ runs:
       japi-compliance-checker ~/.m2/repository/uk/co/gresearch/spark/spark-extension_${{ inputs.scala-compat-version }}/${{ inputs.package-version }}-${{ inputs.spark-compat-version }}/spark-extension_${{ inputs.scala-compat-version }}-${{ inputs.package-version }}-${{ inputs.spark-compat-version }}.jar target/spark-extension*.jar
     shell: bash
 
+  - name: Save Maven packages cache
+    if: github.event_name != 'schedule' && github.ref == 'refs/heads/master'
+    uses: actions/cache/save@v4
+    with:
+      path: ~/.m2/repository
+      key: ${{ runner.os }}-mvn-check-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}-${{ github.run_id }}}
+
   - name: Upload Report
     uses: actions/upload-artifact@v4
     if: always() && steps.exists.outcome == 'success'

diff --git a/.github/actions/test-jvm/action.yml b/.github/actions/test-jvm/action.yml
@@ -37,12 +37,14 @@ runs:
       name: Binaries-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
       path: .
 
-  - name: Cache Spark Binaries
-    uses: actions/cache@v4
-    if: ( ! contains(inputs.spark-version, '-SNAPSHOT') )
+  - name: Restore Spark Binaries cache
+    if: github.event_name != 'schedule' && ! contains(inputs.spark-version, '-SNAPSHOT')
+    uses: actions/cache/restore@v4
     with:
       path: ~/spark
       key: ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}
+      restore-keys:
+        ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}
 
   - name: Setup Spark Binaries
     if: ( ! contains(inputs.spark-version, '-SNAPSHOT') )
@@ -57,13 +59,15 @@ runs:
       echo "SPARK_HOME=$(cd ~/spark; pwd)" >> $GITHUB_ENV
     shell: bash
 
-  - name: Cache Maven packages
-    if: github.event_name != 'merge_group'
-    uses: actions/cache@v4
+  - name: Restore Maven packages cache
+    if: github.event_name != 'schedule'
+    uses: actions/cache/restore@v4
     with:
       path: ~/.m2/repository
       key: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
-      restore-keys: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
+      restore-keys:
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
 
   - name: Setup JDK ${{ inputs.java-compat-version }}
     uses: actions/setup-java@v4

diff --git a/.github/actions/test-python/action.yml b/.github/actions/test-python/action.yml
@@ -43,12 +43,14 @@ runs:
       name: Binaries-${{ inputs.spark-compat-version }}-${{ inputs.scala-compat-version }}
       path: .
 
-  - name: Cache Spark Binaries
-    uses: actions/cache@v4
-    if: inputs.scala-compat-version == '2.12' && ! contains(inputs.spark-version, '-SNAPSHOT')
+  - name: Restore Spark Binaries cache
+    if: github.event_name != 'schedule' && inputs.scala-compat-version == '2.12' && ! contains(inputs.spark-version, '-SNAPSHOT')
+    uses: actions/cache/restore@v4
     with:
       path: ~/spark
       key: ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}
+      restore-keys:
+        ${{ runner.os }}-spark-binaries-${{ inputs.spark-version }}-${{ inputs.scala-compat-version }}
 
   - name: Setup Spark Binaries
     if: inputs.scala-compat-version == '2.12' && ! contains(inputs.spark-version, '-SNAPSHOT')
@@ -63,27 +65,33 @@ runs:
       echo "SPARK_BIN_HOME=$(cd ~/spark; pwd)" >> $GITHUB_ENV
     shell: bash
 
-  - name: Cache Maven packages
-    if: github.event_name != 'merge_group'
-    uses: actions/cache@v4
+  - name: Restore Maven packages cache
+    if: github.event_name != 'schedule'
+    uses: actions/cache/restore@v4
     with:
       path: ~/.m2/repository
       key: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
-      restore-keys: ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
+      restore-keys:
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-${{ hashFiles('pom.xml') }}
+        ${{ runner.os }}-mvn-build-${{ inputs.spark-version }}-${{ inputs.scala-version }}-
 
   - name: Setup JDK 1.8
     uses: actions/setup-java@v4
     with:
       java-version: '8'
       distribution: 'zulu'
 
-  - name: Cache Pip packages
-    if: github.event_name != 'merge_group'
-    uses: actions/cache@v4
+  - name: Restore Pip packages cache
+    if: github.event_name != 'schedule'
+    uses: actions/cache/restore@v4
     with:
       path: ~/.cache/pip
       key: ${{ runner.os }}-pip-test-${{ inputs.python-version }}-${{ hashFiles(format('python/requirements-{0}_{1}.txt', inputs.spark-compat-version, inputs.scala-compat-version)) }}
-      restore-keys: ${{ runner.os }}-pip-test-${{ inputs.python-version }}-
+      restore-keys:
+        ${{ runner.os }}-pip-test-${{ inputs.python-version }}-${{ hashFiles(format('python/requirements-{0}_{1}.txt', inputs.spark-compat-version, inputs.scala-compat-version)) }}
+        ${{ runner.os }}-pip-whl-${{ inputs.python-version }}-${{ hashFiles(format('python/requirements-{0}_{1}.txt', inputs.spark-compat-version, inputs.scala-compat-version)) }}
+        ${{ runner.os }}-pip-test-${{ inputs.python-version }}-
+        ${{ runner.os }}-pip-whl-${{ inputs.python-version }}-
 
   - name: Setup Python
     uses: actions/setup-python@v5
@@ -181,6 +189,13 @@ runs:
       $SPARK_HOME/bin/spark-shell --packages uk.co.gresearch.spark:spark-extension_${{ inputs.scala-compat-version }}:$SPARK_EXTENSION_VERSION < test-release.scala
     shell: bash
 
+  - name: Save Pip packages cache
+    if: github.event_name != 'schedule' && github.ref == 'refs/heads/master'
+    uses: actions/cache/save@v4
+    with:
+      path: ~/.cache/pip
+      key: ${{ runner.os }}-pip-test-${{ inputs.python-version }}-${{ hashFiles(format('python/requirements-{0}_{1}.txt', inputs.spark-compat-version, inputs.scala-compat-version)) }}-${{ github.run_id }}}
+
   - name: Upload Test Results
     if: always()
     uses: actions/upload-artifact@v4