Skip to content

Commit

Permalink
ci: Fix weekly benchmark report
Browse files Browse the repository at this point in the history
  • Loading branch information
@samuelburnham
samuelburnham committed Oct 10, 2024
1 parent fa33d25 commit a65a417
Show file tree
Hide file tree
Showing 2 changed files with 115 additions and 14 deletions.
1 change: 1 addition & 0 deletions .github/actions/setup/action.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ runs:
workspaces: |
aptos -> target
ethereum -> target
kadena -> target
- uses: taiki-e/install-action@nextest
- name: Install deps
shell: bash
Expand Down
128 changes: 114 additions & 14 deletions .github/workflows/bench.yml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Runs benchmarks on self-hosted infra via `workflow_dispatch`
# This trigger can be found at https://github.com/argumentcomputer/zk-light-clients/actions/workflows/bench.yml
# Runs benchmarks on self-hosted infra via `workflow_dispatch` and `schedule`
# The `workflow_dispatch` trigger can be found at https://github.com/argumentcomputer/zk-light-clients/actions/workflows/bench.yml
#
# The benchmark report can be found in the logs and as a comment on the latest commit on `dev`.
# The report can also be sent as a Zulip message to https://zulip.argument.xyz
Expand All @@ -12,13 +12,16 @@ on:
description: 'Name of the light client to benchmark'
type: string
required: true
# Name of the `light-client` benchmark to run, e.g. `inclusion`, `epoch_change` (for Aptos), `committee_change` (for Ethereum)
# Name of the `light-client` benchmark to run:
# Aptos: `inclusion`, `epoch_change`
# Ethereum: `inclusion`, `committee_change`
# Kadena: `spv`, `longest_chain`
# Runs in the `light-client` directory, so it cannot benchmark `proof_server` or `programs`
bench-name:
description: 'Name of the benchmark to run'
description: 'Name of the benchmark to run. See inline docs for examples'
type: string
required: true
# List of comma-separated env vars, e.g. `RUST_LOG=debug,SNARK=1`
# List of comma-separated env vars, e.g. `RUST_LOG=debug,MODE=STARK`
# `RUSTFLAGS="-C target-cpu=native --cfg tokio_unstable -C opt-level=3"` is set by default
env:
description: 'List of comma-separated environment variables'
Expand Down Expand Up @@ -52,17 +55,18 @@ on:
type: string
required: false
schedule:
# Bench report on Tuesdays at 11pm EST/12pm EDT
# Bench report on Tuesdays at 11am EST/12pm EDT
- cron: '0 16 * * 4'

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

jobs:
benchmark:
manual-benchmark:
name: Light client benchmark
runs-on: warp-custom-r7iz-metal-16xl
if: github.event_name == 'workflow_dispatch'
runs-on: warp-custom-r7iz-metal-32xl
steps:
- uses: actions/checkout@v4
with:
Expand All @@ -86,8 +90,9 @@ jobs:
echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV
echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV
echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV
echo "SHARD_CHUNKING_MULTIPLIER=256" | tee -a $GITHUB_ENV
echo "SHARD_CHUNKING_MULTIPLIER=32" | tee -a $GITHUB_ENV
echo "SNARK=1" | tee -a $GITHUB_ENV
echo "MODE=SNARK" | tee -a $GITHUB_ENV
IFS=',' read -ra ENV_VARS <<< "${{ inputs.env }}"
for VAR in "${ENV_VARS[@]}"; do
Expand All @@ -100,7 +105,7 @@ jobs:
if [[ "${{ inputs.zulip }}" == "true" ]]; then
if [[ ! -z "${{ inputs.private }}" ]]; then
TYPE="private"
# Stream = private DM
# Set stream = private DM
STREAM="${{ inputs.private }}"
else
TYPE="stream"
Expand Down Expand Up @@ -134,16 +139,16 @@ jobs:
done < <(paste cycles.txt timings.txt)
echo "# Benchmark Results " | tee -a summary.md
echo "## ${{ inputs.bench-name }} Prove" | tee -a summary.md
echo "## \`${{ inputs.light-client }}\` \`${{ inputs.bench-name }}\` Prove" | tee -a summary.md
cat summary.json | jtbl -m | tee -a summary.md
echo "" | tee -a summary.md
echo "Time unit = milliseconds" | tee -a summary.md
echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md
echo "report<<EOF" >> $GITHUB_OUTPUT
cat summary.md >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
echo "report<<EOF" | tee -a $GITHUB_OUTPUT
cat summary.md | tee -a $GITHUB_OUTPUT
echo "EOF" | tee -a $GITHUB_OUTPUT
working-directory: ${{ github.workspace }}/${{ inputs.light-client }}/light-client
- name: Write bench on commit comment
uses: peter-evans/commit-comment@v3
Expand All @@ -161,3 +166,98 @@ jobs:
# Ignored if `type: private`
topic: "${{ env.TOPIC }}"
content: "${{ steps.run-benchmarks.outputs.report }}"

# Run the weekly benchmark report
weekly-benchmark:
name: Weekly benchmark report
if: github.event_name == 'schedule'
runs-on: ubuntu-latest
runs-on: warp-custom-r7iz-metal-32xl
strategy:
matrix:
light-client: [aptos, ethereum, kadena]
include:
- bench: inclusion
light-client: aptos
- bench: epoch_change
light-client: aptos
- bench: inclusion
light-client: ethereum
- bench: committee_change
light-client: ethereum
- bench: spv
light-client: kadena
- bench: longest_chain
light-client: kadena
steps:
- uses: actions/checkout@v4
with:
repository: argumentcomputer/ci-workflows
- uses: ./.github/actions/ci-env
- uses: actions/checkout@v4
- name: Setup CI
uses: ./.github/actions/setup
- name: Install extra deps
run: |
sudo apt-get update && sudo apt-get install -y python3-pip
pip3 --version
pip3 install --break-system-packages jtbl
echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.profile
source ~/.profile
which jtbl
- name: Set env
run: |
# Default benchmark settings optimized for light clients, can be overwritten with `env` input
echo "RUSTFLAGS=-C target-cpu=native --cfg tokio_unstable -C opt-level=3" | tee -a $GITHUB_ENV
echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV
echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV
echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV
echo "SHARD_CHUNKING_MULTIPLIER=32" | tee -a $GITHUB_ENV
echo "SNARK=1" | tee -a $GITHUB_ENV
echo "MODE=SNARK" | tee -a $GITHUB_ENV
- name: Set Zulip output
run: |
echo "TYPE=stream" | tee -a $GITHUB_ENV
echo "STREAM=light-client" | tee -a $GITHUB_ENV
echo "TOPIC=Benchmark reports" | tee -a $GITHUB_ENV
- name: Run benchmarks
id: run-benchmarks
run: |
make bench-ci BENCH=${{ matrix.bench }} 2>&1 | tee out.txt
grep 'cycles=' out.txt > cycles.txt
grep 'proving_time' out.txt > timings.txt
while IFS=$'\t' read -r f1 f2
do
num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }')
echo "$f2" | jq -c ". += {\"cycles\": $num_cycles}" >> summary.json
done < <(paste cycles.txt timings.txt)
echo "# Benchmark Results " | tee -a summary.md
echo "## \`${{ matrix.light-client }}\` \`${{ matrix.bench }}\` Prove" | tee -a summary.md
cat summary.json | jtbl -m | tee -a summary.md
echo "" | tee -a summary.md
echo "Time unit = milliseconds" | tee -a summary.md
echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md
echo "report<<EOF" | tee -a $GITHUB_OUTPUT
cat summary.md | tee -a $GITHUB_OUTPUT
echo "EOF" | tee -a $GITHUB_OUTPUT
working-directory: ${{ github.workspace }}/${{ matrix.light-client }}/light-client
- name: Write bench on commit comment
uses: peter-evans/commit-comment@v3
with:
body-path: ${{ github.workspace }}/${{ matrix.light-client }}/light-client/summary.md
- name: Send report to Zulip
uses: zulip/github-actions-zulip/send-message@v1
with:
api-key: ${{ secrets.ZULIP_API_KEY }}
email: "[email protected]"
organization-url: "https://zulip.lurk-lab.com"
to: "${{ env.STREAM }}"
type: "${{ env.TYPE }}"
# Ignored if `type: private`
topic: "${{ env.TOPIC }}"
content: "${{ steps.run-benchmarks.outputs.report }}"

0 comments on commit a65a417

Please sign in to comment.