Test

.github/workflows/bench.yml

@@ -5,14 +5,19 @@
 # The report can also be sent as a Zulip message to https://zulip.argument.xyz
 name: Light client benchmark
 on:
+  push: 
+    branches: ci-bench
   workflow_dispatch:
     inputs:
       # Which light client to bench, e.g. `aptos`, `ethereum` or `kadena`
       light-client:
         description: 'Name of the light client to benchmark'
         type: string
         required: true
-      # Name of the `light-client` benchmark to run, e.g. `inclusion`, `epoch_change` (for Aptos), `committee_change` (for Ethereum)
+      # Name of the `light-client` benchmark to run, e.g.
+      # Aptos: `inclusion` or `epoch_change`
+      # Ethereum: `inclusion or `committee_change`
+      # Kadena: `longest_chain` or `spv`
       # Runs in the `light-client` directory, so it cannot benchmark `proof_server` or `programs`
       bench-name:
         description: 'Name of the benchmark to run'
@@ -62,6 +67,7 @@ concurrency:
 jobs:
   benchmark:
     name: Light client benchmark
+    if: github.event_name == 'workflow_dispatch'
     runs-on: warp-custom-r7iz-metal-16xl
     steps:
       - uses: actions/checkout@v4
@@ -155,7 +161,89 @@ jobs:
         with:
           api-key: ${{ secrets.ZULIP_API_KEY }}
           email: "[email protected]"
-          organization-url: "https://zulip.lurk-lab.com"
+          organization-url: "https://zulip.argument.xyz"
+          to: "${{ env.STREAM }}"
+          type: "${{ env.TYPE }}"
+          # Ignored if `type: private`
+          topic: "${{ env.TOPIC }}"
+          content: "${{ steps.run-benchmarks.outputs.report }}"
+
+  bench-test:
+    name: Light client benchmark test
+    runs-on: warp-custom-r7iz-metal-16xl
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: argumentcomputer/ci-workflows
+      - uses: ./.github/actions/ci-env
+      - uses: actions/checkout@v4
+      - name: Setup CI
+        uses: ./.github/actions/setup
+      - name: Install extra deps
+        run: |
+          sudo apt-get update && sudo apt-get install -y python3-pip
+          pip3 --version
+          pip3 install --break-system-packages jtbl
+          echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.profile
+          source ~/.profile
+          which jtbl
+      - name: Set env
+        run: |
+          # Default benchmark settings optimized for light clients, can be overwritten with `env` input
+          echo "RUSTFLAGS=-C target-cpu=native --cfg tokio_unstable -C opt-level=3" | tee -a $GITHUB_ENV
+          echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV
+          echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV
+          echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV
+          echo "SHARD_CHUNKING_MULTIPLIER=256" | tee -a $GITHUB_ENV
+          echo "SNARK=1" | tee -a $GITHUB_ENV
+
+          IFS=',' read -ra ENV_VARS <<< "${{ inputs.env }}"
+          for VAR in "${ENV_VARS[@]}"; do
+            VAR_NAME="${VAR%%=*}"
+            VAR_VALUE="${VAR#*=}"
+            echo "${VAR_NAME}=${VAR_VALUE}" | tee -a $GITHUB_ENV
+          done
+      - name: Parse Zulip inputs
+        run: |
+          echo "TYPE=private" | tee -a $GITHUB_ENV
+          echo "STREAM=11" | tee -a $GITHUB_ENV
+      - name: Run benchmarks
+        id: run-benchmarks
+        run: |
+          make bench-ci BENCH=${{ inputs.bench-name }} 2>&1 | tee out.txt
+
+          grep 'cycles=' out.txt > cycles.txt
+          grep 'proving_time' out.txt > timings.txt
+
+          while IFS=$'\t' read -r f1 f2
+          do
+            num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }')
+            echo "$f2" | jq -c ". += {\"cycles\": $num_cycles}" >> summary.json
+          done < <(paste cycles.txt timings.txt)
+
+          echo "# Benchmark Results " | tee -a summary.md
+          echo "## ${{ inputs.bench-name }} Prove" | tee -a summary.md
+          cat summary.json | jtbl -m | tee -a summary.md
+          echo "" | tee -a summary.md
+
+          echo "Time unit = milliseconds" | tee -a summary.md
+          echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md
+
+          echo "report<<EOF" >> $GITHUB_OUTPUT
+          cat summary.md >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+        working-directory: ${{ github.workspace }}/${{ inputs.light-client }}/light-client
+      #- name: Write bench on commit comment
+      #  uses: peter-evans/commit-comment@v3
+      #  with:
+      #    body-path: ${{ github.workspace }}/${{ inputs.light-client }}/light-client/summary.md
+      - name: Send report to Zulip
+        if: inputs.zulip
+        uses: zulip/github-actions-zulip/send-message@v1
+        with:
+          api-key: ${{ secrets.ZULIP_API_KEY }}
+          email: "[email protected]"
+          organization-url: "https://zulip.argument.xyz"
           to: "${{ env.STREAM }}"
           type: "${{ env.TYPE }}"
           # Ignored if `type: private`