ci: Add automated benchmark report (#302)

* ci: Use custom runner for `bench.yml` * Test * Test epoch_change * Test ethereum inclusion * Test Kadena SPV * Add scheduled workflow * fix: add logger to retrieve cycle count * Tidying * perf: fix bench env * refactor: refactor env in workflow * test: test workflow and bench changes * refactor: refactor schedule * test: test on 32xl * refactor: remove test + pre download plonk artifacts if needed * refactor: proper inline comment + remove test workflow trigger --------- Co-authored-by: Thomas Chataigner <[email protected]>
argumentcomputer · Oct 30, 2024 · 034f0c0 · 034f0c0 · github-actions · Oct 31, 2024
1 parent 00283b8
commit 034f0c0
Show file tree

Hide file tree

Showing 12 changed files with 205 additions and 25 deletions.
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
@@ -26,6 +26,7 @@ runs:
         workspaces: |
           aptos -> target
           ethereum -> target
+          kadena -> target
     - uses: taiki-e/install-action@nextest
     - name: Install deps
       shell: bash

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
@@ -1,24 +1,35 @@
-# Runs benchmarks on self-hosted infra via `workflow_dispatch`
-# This trigger can be found at https://github.com/argumentcomputer/zk-light-clients/actions/workflows/bench.yml
+# Runs benchmarks on self-hosted infra via `workflow_dispatch` and scheduled on Mondays at 9am EST
 #
+# `workflow_dispatch` trigger
+# The workflow can be run at https://github.com/argumentcomputer/zk-light-clients/actions/workflows/bench.yml
 # The benchmark report can be found in the logs and as a comment on the latest commit on `dev`.
 # The report can also be sent as a Zulip message to https://zulip.argument.xyz
+#
+# `schedule` trigger
+# The workflow runs every week on Monday at 9am EST
+# It runs all light client benchmarks and sends them to Zulip
 name: Light client benchmark
 on:
+  schedule:
+    # Full report on Thursday, 2pm UTC
+    - cron: '0 14 * * 4'
   workflow_dispatch:
     inputs:
       # Which light client to bench, e.g. `aptos`, `ethereum` or `kadena`
       light-client:
         description: 'Name of the light client to benchmark'
         type: string
         required: true
-      # Name of the `light-client` benchmark to run, e.g. `inclusion`, `epoch_change` (for Aptos), `committee_change` (for Ethereum)
+      # Name of the `light-client` benchmark to run, e.g.
+      # Aptos: `inclusion` or `epoch_change`
+      # Ethereum: `inclusion or `committee_change`
+      # Kadena: `longest_chain` or `spv`
       # Runs in the `light-client` directory, so it cannot benchmark `proof_server` or `programs`
       bench-name:
         description: 'Name of the benchmark to run'
         type: string
         required: true
-      # List of comma-separated env vars, e.g. `RUST_LOG=debug,SNARK=1`
+      # List of comma-separated env vars, e.g. `RUST_LOG=debug,MODE=SNARK`
       # `RUSTFLAGS="-C target-cpu=native --cfg tokio_unstable -C opt-level=3"` is set by default
       env:
         description: 'List of comma-separated environment variables'
@@ -40,29 +51,27 @@ on:
       # Zulip stream in which to send the message (optional)
       # Ignored if `private` input is specified
       # Defaults to `light_client` stream
-      stream:
-        description: 'Send message to stream (default is `light-client`). Ignored if DM'
+      channel:
+        description: 'Send message to channel (default is `light-client`). Ignored if `private` input is specified'
         type: string
         required: false
       # Zulip topic in which to send the message (optional)
       # Ignored if `private` input is specified
       # Defaults to `chat`
       topic:
-        description: 'Send message to topic (default is `chat`). Ignored if DM'
+        description: 'Send message to topic (default is `chat`). Ignored if `private` input is specified'
         type: string
         required: false
-  schedule:
-    # Bench report on Tuesdays at 11pm EST/12pm EDT
-    - cron: '0 16 * * 4'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 jobs:
-  benchmark:
-    name: Light client benchmark
-    runs-on: warp-custom-r7iz-metal-16xl
+  benchmark-manual:
+    name: Light client benchmark (manual)
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: warp-custom-r7iz-metal-32xl
     steps:
       - uses: actions/checkout@v4
         with:
@@ -86,8 +95,8 @@ jobs:
           echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV
           echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV
           echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV
-          echo "SHARD_CHUNKING_MULTIPLIER=256" | tee -a $GITHUB_ENV
-          echo "SNARK=1" | tee -a $GITHUB_ENV
+          echo "SHARD_CHUNKING_MULTIPLIER=1" | tee -a $GITHUB_ENV
+          echo "MODE=SNARK" | tee -a $GITHUB_ENV
 
           IFS=',' read -ra ENV_VARS <<< "${{ inputs.env }}"
           for VAR in "${ENV_VARS[@]}"; do
@@ -120,25 +129,38 @@ jobs:
             echo "TOPIC=$TOPIC" | tee -a $GITHUB_ENV
           fi
       - name: Run benchmarks
-        id: run-benchmarks
         run: |
           make bench-ci BENCH=${{ inputs.bench-name }} 2>&1 | tee out.txt
-
+        working-directory: ${{ github.workspace }}/${{ inputs.light-client }}/light-client
+      - name: Create report
+        id: run-benchmarks
+        run: |
           grep 'cycles=' out.txt > cycles.txt
           grep 'proving_time' out.txt > timings.txt
 
           while IFS=$'\t' read -r f1 f2
           do
             num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }')
-            echo "$f2" | jq -c ". += {\"cycles\": $num_cycles}" >> summary.json
+            timings=$(echo "$f2" | jq '
+              to_entries |
+              map(
+                if .key == "proving_time" or .key == "verifying_time" then
+                  {key, value: (.value / 1000 | floor as $s | "\(($s / 60 | floor) | tostring)min\(($s % 60) | tostring)s")}
+                else
+                  .
+                end
+              ) |
+              from_entries
+            ')
+
+            echo "$timings" | jq -c --argjson cycles "$num_cycles" '. += {cycles: $cycles}' >> summary.json
           done < <(paste cycles.txt timings.txt)
 
-          echo "# Benchmark Results " | tee -a summary.md
-          echo "## ${{ inputs.bench-name }} Prove" | tee -a summary.md
+          echo '# `${{ inputs.light-client }}` Benchmark Results' | tee -a summary.md
+          echo '## `${{ inputs.bench-name }}` Prove' | tee -a summary.md
           cat summary.json | jtbl -m | tee -a summary.md
           echo "" | tee -a summary.md
 
-          echo "Time unit = milliseconds" | tee -a summary.md
           echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md
 
           echo "report<<EOF" >> $GITHUB_OUTPUT
@@ -155,9 +177,122 @@ jobs:
         with:
           api-key: ${{ secrets.ZULIP_API_KEY }}
           email: "[email protected]"
-          organization-url: "https://zulip.lurk-lab.com"
+          organization-url: "https://zulip.argument.xyz"
           to: "${{ env.STREAM }}"
           type: "${{ env.TYPE }}"
           # Ignored if `type: private`
           topic: "${{ env.TOPIC }}"
           content: "${{ steps.run-benchmarks.outputs.report }}"
+
+  benchmark-scheduled:
+    name: Light client bench (scheduled)
+    if: github.event_name == 'schedule'
+    runs-on: warp-custom-r7iz-metal-32xl
+    strategy:
+      fail-fast: false
+      matrix:
+        light-client: [ aptos, ethereum, kadena ]
+        include:
+          - benchmark: inclusion
+            light-client: aptos
+          - benchmark: epoch_change
+            light-client: aptos
+          - benchmark: inclusion
+            light-client: ethereum
+          - benchmark: committee_change
+            light-client: ethereum
+          - benchmark: spv
+            light-client: kadena
+          - benchmark: longest_chain
+            light-client: kadena
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: argumentcomputer/ci-workflows
+      - uses: ./.github/actions/ci-env
+      - uses: actions/checkout@v4
+      - name: Setup CI
+        uses: ./.github/actions/setup
+      - name: Install extra deps
+        run: |
+          sudo apt-get update && sudo apt-get install -y python3-pip
+          pip3 --version
+          pip3 install --break-system-packages jtbl
+          echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.profile
+          source ~/.profile
+          which jtbl
+      - name: Set env
+        run: |
+          # Default benchmark settings optimized for light clients, can be overwritten with `env` input
+          echo "RUSTFLAGS=-C target-cpu=native --cfg tokio_unstable -C opt-level=3" | tee -a $GITHUB_ENV
+          echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV
+          echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV
+          echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV
+          echo "SHARD_CHUNKING_MULTIPLIER=1" | tee -a $GITHUB_ENV
+          echo "MODE=SNARK" | tee -a $GITHUB_ENV
+
+          IFS=',' read -ra ENV_VARS <<< "ethereum"
+          for VAR in "${ENV_VARS[@]}"; do
+            VAR_NAME="${VAR%%=*}"
+            VAR_VALUE="${VAR#*=}"
+            echo "${VAR_NAME}=${VAR_VALUE}" | tee -a $GITHUB_ENV
+          done
+      - name: Set Zulip env
+        run: |
+          echo "TYPE=stream" | tee -a $GITHUB_ENV
+          echo "STREAM=light-client" | tee -a $GITHUB_ENV
+          echo "TOPIC=Benchmark Reports" | tee -a $GITHUB_ENV
+      - name: Run benchmarks
+        run: |
+          make bench-ci BENCH=${{ matrix.benchmark }} 2>&1 | tee out.txt
+        working-directory: ${{ github.workspace }}/${{ matrix.light-client }}/light-client
+      - name: Create report
+        id: run-benchmarks
+        run: |
+          grep 'cycles=' out.txt > cycles.txt
+          grep 'proving_time' out.txt > timings.txt
+
+          while IFS=$'\t' read -r f1 f2
+          do
+            num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }')
+            timings=$(echo "$f2" | jq '
+              to_entries |
+              map(
+                if .key == "proving_time" or .key == "verifying_time" then
+                  {key, value: (.value / 1000 | floor as $s | "\(($s / 60 | floor) | tostring)min\(($s % 60) | tostring)s")}
+                else
+                  .
+                end
+              ) |
+              from_entries
+            ')
+
+            echo "$timings" | jq -c --argjson cycles "$num_cycles" '. += {cycles: $cycles}' >> summary.json
+          done < <(paste cycles.txt timings.txt)
+
+          echo '# `${{ matrix.light-client }}` Benchmark Results' | tee -a summary.md
+          echo '## `${{ matrix.benchmark }}` Prove' | tee -a summary.md
+          cat summary.json | jtbl -m | tee -a summary.md
+          echo "" | tee -a summary.md
+
+          echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md
+
+          echo "report<<EOF" >> $GITHUB_OUTPUT
+          cat summary.md >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+        working-directory: ${{ github.workspace }}/${{ matrix.light-client }}/light-client
+      - name: Write bench on commit comment
+        uses: peter-evans/commit-comment@v3
+        with:
+          body-path: ${{ github.workspace }}/${{ matrix.light-client }}/light-client/summary.md
+      - name: Send report to Zulip
+        uses: zulip/github-actions-zulip/send-message@v1
+        with:
+          api-key: ${{ secrets.ZULIP_API_KEY }}
+          email: "[email protected]"
+          organization-url: "https://zulip.argument.xyz"
+          to: "${{ env.STREAM }}"
+          type: "${{ env.TYPE }}"
+          # Ignored if `type: private`
+          topic: "${{ env.TOPIC }}"
+          content: "${{ steps.run-benchmarks.outputs.report }}"
diff --git a/aptos/light-client/Makefile b/aptos/light-client/Makefile
@@ -5,7 +5,7 @@ benchmark:
 	RUSTFLAGS="--cfg tokio_unstable -C target-cpu=native -C opt-level=3" \
 	SHARD_SIZE=4194304 \
 	SHARD_BATCH_SIZE=0 \
-	SHARD_CHUNKING_MULTIPLIER=64 \
+	SHARD_CHUNKING_MULTIPLIER=1 \
 	RECONSTRUCT_COMMITMENTS=false \
 	cargo bench --features aptos --bench $$bench
 

diff --git a/aptos/light-client/benches/epoch_change.rs b/aptos/light-client/benches/epoch_change.rs
@@ -21,6 +21,7 @@ use aptos_lc_core::aptos_test_utils::wrapper::AptosWrapper;
 use aptos_lc_core::crypto::hash::CryptoHash;
 use aptos_lc_core::types::trusted_state::TrustedState;
 use serde::Serialize;
+use sphinx_sdk::artifacts::try_install_plonk_bn254_artifacts;
 use sphinx_sdk::utils::setup_logger;
 use sphinx_sdk::{ProverClient, SphinxProofWithPublicValues, SphinxStdin};
 use std::env;
@@ -131,6 +132,10 @@ fn main() {
     // Initialize the proving assets and benchmark the proving process.
     let proving_assets = ProvingAssets::new(mode);
 
+    if mode == ProvingMode::SNARK {
+        let _ = try_install_plonk_bn254_artifacts(false);
+    }
+
     let start_proving = Instant::now();
     let mut epoch_change_proof = proving_assets.prove();
     let proving_time = start_proving.elapsed();

diff --git a/aptos/light-client/benches/inclusion.rs b/aptos/light-client/benches/inclusion.rs
@@ -27,6 +27,7 @@ use aptos_lc_core::types::ledger_info::LedgerInfoWithSignatures;
 use aptos_lc_core::types::trusted_state::TrustedState;
 use aptos_lc_core::types::validator::ValidatorVerifier;
 use serde::Serialize;
+use sphinx_sdk::artifacts::try_install_plonk_bn254_artifacts;
 use sphinx_sdk::utils::setup_logger;
 use sphinx_sdk::{ProverClient, SphinxProofWithPublicValues, SphinxStdin};
 use std::env;
@@ -173,6 +174,11 @@ struct Timings {
 fn main() {
     let mode_str: String = env::var("MODE").unwrap_or_else(|_| "STARK".into());
     let mode = ProvingMode::try_from(mode_str.as_str()).expect("MODE should be STARK or SNARK");
+
+    if mode == ProvingMode::SNARK {
+        let _ = try_install_plonk_bn254_artifacts(false);
+    }
+
     for nbr_leaves in NBR_LEAVES {
         let proving_assets = ProvingAssets::from_nbr_leaves(mode, nbr_leaves);
 

diff --git a/aptos/light-client/benches/sig.rs b/aptos/light-client/benches/sig.rs
@@ -4,6 +4,7 @@
 use anyhow::anyhow;
 use aptos_lc_core::aptos_test_utils::wrapper::AptosWrapper;
 use serde::Serialize;
+use sphinx_sdk::artifacts::try_install_plonk_bn254_artifacts;
 use sphinx_sdk::utils::setup_logger;
 use sphinx_sdk::{ProverClient, SphinxProofWithPublicValues, SphinxStdin};
 use std::env;
@@ -99,6 +100,10 @@ fn main() {
 
     let proving_assets = ProvingAssets::new(mode);
 
+    if mode == ProvingMode::SNARK {
+        let _ = try_install_plonk_bn254_artifacts(false);
+    }
+
     let start_proving = Instant::now();
     let proof = proving_assets.prove();
     let proving_time = start_proving.elapsed();

diff --git a/ethereum/light-client/Makefile b/ethereum/light-client/Makefile
@@ -5,7 +5,7 @@ benchmark:
 	RUSTFLAGS="-C target-cpu=native -C opt-level=3" \
 	SHARD_SIZE=4194304 \
 	SHARD_BATCH_SIZE=0 \
-	SHARD_CHUNKING_MULTIPLIER=64 \
+	SHARD_CHUNKING_MULTIPLIER=1 \
 	RECONSTRUCT_COMMITMENTS=false \
 	cargo bench --features ethereum --bench $$bench
 

diff --git a/ethereum/light-client/benches/committee_change.rs b/ethereum/light-client/benches/committee_change.rs
@@ -7,6 +7,7 @@ use ethereum_lc_core::types::bootstrap::Bootstrap;
 use ethereum_lc_core::types::store::LightClientStore;
 use ethereum_lc_core::types::update::Update;
 use serde::Serialize;
+use sphinx_sdk::artifacts::try_install_plonk_bn254_artifacts;
 use std::env::current_dir;
 use std::time::Instant;
 use std::{env, fs};
@@ -74,9 +75,15 @@ struct BenchResults {
 }
 
 fn main() {
+    sphinx_sdk::utils::setup_logger();
+
     let mode_str: String = env::var("MODE").unwrap_or_else(|_| "STARK".into());
     let mode = ProvingMode::try_from(mode_str.as_str()).expect("MODE should be STARK or SNARK");
 
+    if mode == ProvingMode::SNARK {
+        let _ = try_install_plonk_bn254_artifacts(false);
+    }
+
     // Instantiate BenchmarkAssets
     let mut benchmark_assets = BenchmarkAssets::generate();
 

diff --git a/ethereum/light-client/benches/inclusion.rs b/ethereum/light-client/benches/inclusion.rs
@@ -9,6 +9,7 @@ use ethereum_lc_core::types::bootstrap::Bootstrap;
 use ethereum_lc_core::types::store::LightClientStore;
 use ethereum_lc_core::types::update::{FinalityUpdate, Update};
 use serde::Serialize;
+use sphinx_sdk::artifacts::try_install_plonk_bn254_artifacts;
 use std::env::current_dir;
 use std::time::Instant;
 use std::{env, fs};
@@ -87,9 +88,15 @@ struct BenchResults {
 }
 
 fn main() {
+    sphinx_sdk::utils::setup_logger();
+
     let mode_str: String = env::var("MODE").unwrap_or_else(|_| "STARK".into());
     let mode = ProvingMode::try_from(mode_str.as_str()).expect("MODE should be STARK or SNARK");
 
+    if mode == ProvingMode::SNARK {
+        let _ = try_install_plonk_bn254_artifacts(false);
+    }
+
     // Instantiate BenchmarkAssets
     let benchmark_assets = BenchmarkAssets::generate();
 

diff --git a/kadena/light-client/Makefile b/kadena/light-client/Makefile
@@ -5,7 +5,7 @@ benchmark:
 	RUSTFLAGS="-C target-cpu=native -C opt-level=3" \
 	SHARD_SIZE=4194304 \
 	SHARD_BATCH_SIZE=0 \
-	SHARD_CHUNKING_MULTIPLIER=64 \
+	SHARD_CHUNKING_MULTIPLIER=1 \
 	RECONSTRUCT_COMMITMENTS=false \
 	cargo bench --features kadena --bench $$bench