Skip to content
Light client benchmark

test: test on 32xl #48

Sign in to view logs

test: test on 32xl

test: test on 32xl #48

Workflow file for this run

.github/workflows/bench.yml at 4f7d491
# Runs benchmarks on self-hosted infra via `workflow_dispatch` and scheduled on Mondays at 9am EST
#
# `workflow_dispatch` trigger
# The workflow can be run at https://github.com/argumentcomputer/zk-light-clients/actions/workflows/bench.yml
# The benchmark report can be found in the logs and as a comment on the latest commit on `dev`.
# The report can also be sent as a Zulip message to https://zulip.argument.xyz
#
# `schedule` trigger
# The workflow runs every week on Monday at 9am EST
# It runs all light client benchmarks and sends them to Zulip
name: Light client benchmark
on:
push:
branches: ci-bench
schedule:
# Full report on Mondays at 9am EST/10am EDT
- cron: '0 14 * * 4'
workflow_dispatch:
inputs:
# Which light client to bench, e.g. `aptos`, `ethereum` or `kadena`
light-client:
description: 'Name of the light client to benchmark'
type: string
required: true
# Name of the `light-client` benchmark to run, e.g.
# Aptos: `inclusion` or `epoch_change`
# Ethereum: `inclusion or `committee_change`
# Kadena: `longest_chain` or `spv`
# Runs in the `light-client` directory, so it cannot benchmark `proof_server` or `programs`
bench-name:
description: 'Name of the benchmark to run'
type: string
required: true
# List of comma-separated env vars, e.g. `RUST_LOG=debug,MODE=SNARK`
# `RUSTFLAGS="-C target-cpu=native --cfg tokio_unstable -C opt-level=3"` is set by default
env:
description: 'List of comma-separated environment variables'
type: string
required: false
# Optionally send a message to the below Zulip streams
# Defaults to false
zulip:
description: 'Send the report to Zulip'
type: boolean
required: false
# User(s) to whom to send a private DM (optional)
# Comma-separated list of user ID integers, e.g. `11,12` (IDs can be found in user profiles)
# If not specified, sends to a stream/topic pair instead
private:
description: 'Send DM to given user ID(s)'
type: string
required: false
# Zulip stream in which to send the message (optional)
# Ignored if `private` input is specified
# Defaults to `light_client` stream
channel:
description: 'Send message to channel (default is `light-client`). Ignored if `private` input is specified'
type: string
required: false
# Zulip topic in which to send the message (optional)
# Ignored if `private` input is specified
# Defaults to `chat`
topic:
description: 'Send message to topic (default is `chat`). Ignored if `private` input is specified'
type: string
required: false
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
benchmark-manual:
name: Light client benchmark (manual)
if: github.event_name == 'workflow_dispatch'
runs-on: warp-custom-r7iz-metal-32xl
steps:
- uses: actions/checkout@v4
with:
repository: argumentcomputer/ci-workflows
- uses: ./.github/actions/ci-env
- uses: actions/checkout@v4
- name: Setup CI
uses: ./.github/actions/setup
- name: Install extra deps
run: |
sudo apt-get update && sudo apt-get install -y python3-pip
pip3 --version
pip3 install --break-system-packages jtbl
echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.profile
source ~/.profile
which jtbl
- name: Set env
run: |
# Default benchmark settings optimized for light clients, can be overwritten with `env` input
echo "RUSTFLAGS=-C target-cpu=native --cfg tokio_unstable -C opt-level=3" | tee -a $GITHUB_ENV
echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV
echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV
echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV
echo "SHARD_CHUNKING_MULTIPLIER=1" | tee -a $GITHUB_ENV
echo "MODE=SNARK" | tee -a $GITHUB_ENV
IFS=',' read -ra ENV_VARS <<< "${{ inputs.env }}"
for VAR in "${ENV_VARS[@]}"; do
VAR_NAME="${VAR%%=*}"
VAR_VALUE="${VAR#*=}"
echo "${VAR_NAME}=${VAR_VALUE}" | tee -a $GITHUB_ENV
done
- name: Parse Zulip inputs
run: |
if [[ "${{ inputs.zulip }}" == "true" ]]; then
if [[ ! -z "${{ inputs.private }}" ]]; then
TYPE="private"
# Stream = private DM
STREAM="${{ inputs.private }}"
else
TYPE="stream"
if [[ ! -z "${{ inputs.stream }}" ]]; then
STREAM="${{ inputs.stream }}"
elif [[ -z "$STREAM" ]]; then
STREAM="light_client"
fi
if [[ ! -z "${{ inputs.topic }}" ]]; then
TOPIC="${{ inputs.topic }}"
elif [[ -z "$TOPIC" ]]; then
TOPIC="chat"
fi
fi
echo "TYPE=$TYPE" | tee -a $GITHUB_ENV
echo "STREAM=$STREAM" | tee -a $GITHUB_ENV
echo "TOPIC=$TOPIC" | tee -a $GITHUB_ENV
fi
- name: Run benchmarks
run: |
make bench-ci BENCH=${{ inputs.bench-name }} 2>&1 | tee out.txt
working-directory: ${{ github.workspace }}/${{ inputs.light-client }}/light-client
- name: Create report
id: run-benchmarks
run: |
grep 'cycles=' out.txt > cycles.txt
grep 'proving_time' out.txt > timings.txt
while IFS=$'\t' read -r f1 f2
do
num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }')
timings=$(echo "$f2" | jq '
to_entries |
map(
if .key == "proving_time" or .key == "verifying_time" then
{key, value: (.value / 1000 | floor as $s | "\(($s / 60 | floor) | tostring)min\(($s % 60) | tostring)s")}
else
.
end
) |
from_entries
')
echo "$timings" | jq -c --argjson cycles "$num_cycles" '. += {cycles: $cycles}' >> summary.json
done < <(paste cycles.txt timings.txt)
echo '# `${{ inputs.light-client }}` Benchmark Results' | tee -a summary.md
echo '## `${{ inputs.bench-name }}` Prove' | tee -a summary.md
cat summary.json | jtbl -m | tee -a summary.md
echo "" | tee -a summary.md
echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md
echo "report<<EOF" >> $GITHUB_OUTPUT
cat summary.md >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
working-directory: ${{ github.workspace }}/${{ inputs.light-client }}/light-client
- name: Write bench on commit comment
uses: peter-evans/commit-comment@v3
with:
body-path: ${{ github.workspace }}/${{ inputs.light-client }}/light-client/summary.md
- name: Send report to Zulip
if: inputs.zulip
uses: zulip/github-actions-zulip/send-message@v1
with:
api-key: ${{ secrets.ZULIP_API_KEY }}
email: "[email protected]"
organization-url: "https://zulip.argument.xyz"
to: "${{ env.STREAM }}"
type: "${{ env.TYPE }}"
# Ignored if `type: private`
topic: "${{ env.TOPIC }}"
content: "${{ steps.run-benchmarks.outputs.report }}"
benchmark-scheduled:
name: Light client bench (scheduled)
if: github.event_name == 'schedule'
runs-on: warp-custom-r7iz-metal-32xl
strategy:
fail-fast: false
matrix:
light-client: [ aptos, ethereum, kadena ]
include:
- benchmark: inclusion
light-client: aptos
- benchmark: epoch_change
light-client: aptos
- benchmark: inclusion
light-client: ethereum
- benchmark: committee_change
light-client: ethereum
- benchmark: spv
light-client: kadena
- benchmark: longest_chain
light-client: kadena
steps:
- uses: actions/checkout@v4
with:
repository: argumentcomputer/ci-workflows
- uses: ./.github/actions/ci-env
- uses: actions/checkout@v4
- name: Setup CI
uses: ./.github/actions/setup
- name: Install extra deps
run: |
sudo apt-get update && sudo apt-get install -y python3-pip
pip3 --version
pip3 install --break-system-packages jtbl
echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.profile
source ~/.profile
which jtbl
- name: Set env
run: |
# Default benchmark settings optimized for light clients, can be overwritten with `env` input
echo "RUSTFLAGS=-C target-cpu=native --cfg tokio_unstable -C opt-level=3" | tee -a $GITHUB_ENV
echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV
echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV
echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV
echo "SHARD_CHUNKING_MULTIPLIER=1" | tee -a $GITHUB_ENV
echo "MODE=SNARK" | tee -a $GITHUB_ENV
IFS=',' read -ra ENV_VARS <<< "ethereum"
for VAR in "${ENV_VARS[@]}"; do
VAR_NAME="${VAR%%=*}"
VAR_VALUE="${VAR#*=}"
echo "${VAR_NAME}=${VAR_VALUE}" | tee -a $GITHUB_ENV
done
- name: Set Zulip env
run: |
echo "TYPE=stream" | tee -a $GITHUB_ENV
echo "STREAM=light-client" | tee -a $GITHUB_ENV
echo "TOPIC=Benchmark Reports" | tee -a $GITHUB_ENV
- name: Run benchmarks
run: |
make bench-ci BENCH=${{ matrix.benchmark }} 2>&1 | tee out.txt
working-directory: ${{ github.workspace }}/${{ matrix.light-client }}/light-client
- name: Create report
id: run-benchmarks
run: |
grep 'cycles=' out.txt > cycles.txt
grep 'proving_time' out.txt > timings.txt
while IFS=$'\t' read -r f1 f2
do
num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }')
timings=$(echo "$f2" | jq '
to_entries |
map(
if .key == "proving_time" or .key == "verifying_time" then
{key, value: (.value / 1000 | floor as $s | "\(($s / 60 | floor) | tostring)min\(($s % 60) | tostring)s")}
else
.
end
) |
from_entries
')
echo "$timings" | jq -c --argjson cycles "$num_cycles" '. += {cycles: $cycles}' >> summary.json
done < <(paste cycles.txt timings.txt)
echo '# `${{ matrix.light-client }}` Benchmark Results' | tee -a summary.md
echo '## `${{ matrix.benchmark }}` Prove' | tee -a summary.md
cat summary.json | jtbl -m | tee -a summary.md
echo "" | tee -a summary.md
echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md
echo "report<<EOF" >> $GITHUB_OUTPUT
cat summary.md >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
working-directory: ${{ github.workspace }}/${{ matrix.light-client }}/light-client
- name: Write bench on commit comment
uses: peter-evans/commit-comment@v3
with:
body-path: ${{ github.workspace }}/${{ matrix.light-client }}/light-client/summary.md
- name: Send report to Zulip
uses: zulip/github-actions-zulip/send-message@v1
with:
api-key: ${{ secrets.ZULIP_API_KEY }}
email: "[email protected]"
organization-url: "https://zulip.argument.xyz"
to: "${{ env.STREAM }}"
type: "${{ env.TYPE }}"
# Ignored if `type: private`
topic: "${{ env.TOPIC }}"
content: "${{ steps.run-benchmarks.outputs.report }}"
bench-test:
name: Light client benchmark test
runs-on: warp-custom-r7iz-metal-32xl
steps:
- uses: actions/checkout@v4
with:
repository: argumentcomputer/ci-workflows
- uses: ./.github/actions/ci-env
- uses: actions/checkout@v4
- name: Setup CI
uses: ./.github/actions/setup
- name: Install extra deps
run: |
sudo apt-get update && sudo apt-get install -y python3-pip
pip3 --version
pip3 install --break-system-packages jtbl
echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.profile
source ~/.profile
which jtbl
- name: Set env
run: |
# Default benchmark settings optimized for light clients, can be overwritten with `env` input
echo "RUSTFLAGS=-C target-cpu=native --cfg tokio_unstable -C opt-level=3" | tee -a $GITHUB_ENV
echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV
echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV
echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV
echo "SHARD_CHUNKING_MULTIPLIER=1" | tee -a $GITHUB_ENV
echo "MODE=SNARK" | tee -a $GITHUB_ENV
IFS=',' read -ra ENV_VARS <<< "ethereum"
for VAR in "${ENV_VARS[@]}"; do
VAR_NAME="${VAR%%=*}"
VAR_VALUE="${VAR#*=}"
echo "${VAR_NAME}=${VAR_VALUE}" | tee -a $GITHUB_ENV
done
- name: Parse Zulip inputs
run: |
echo "TYPE=private" | tee -a $GITHUB_ENV
echo "STREAM=114" | tee -a $GITHUB_ENV
- name: Run benchmarks
id: run-benchmarks
run: |
make bench-ci BENCH=spv 2>&1 | tee out.txt
grep 'cycles=' out.txt > cycles.txt
grep 'proving_time' out.txt > timings.txt
while IFS=$'\t' read -r f1 f2
do
num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }')
echo "$f2" | jq -c --argjson cycles "$num_cycles" '. += {cycles: $cycles}' >> summary.json
done < <(paste cycles.txt timings.txt)
echo "# Kadena Benchmark Results " | tee -a summary.md
echo "## SPV Prove" | tee -a summary.md
cat summary.json | jtbl -m | tee -a summary.md
echo "" | tee -a summary.md
echo "Time unit = milliseconds" | tee -a summary.md
echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md
echo "report<<EOF" >> $GITHUB_OUTPUT
cat summary.md >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
working-directory: ${{ github.workspace }}/kadena/light-client
#- name: Write bench on commit comment
# uses: peter-evans/commit-comment@v3
# with:
# body-path: ${{ github.workspace }}/${{ inputs.light-client }}/light-client/summary.md
- name: Send report to Zulip
uses: zulip/github-actions-zulip/send-message@v1
with:
api-key: ${{ secrets.ZULIP_API_KEY }}
email: "[email protected]"
organization-url: "https://zulip.argument.xyz"
to: "${{ env.STREAM }}"
type: "${{ env.TYPE }}"
# Ignored if `type: private`
topic: "${{ env.TOPIC }}"
content: "${{ steps.run-benchmarks.outputs.report }}"