bench.yml

# Runs benchmarks on self-hosted infra via `workflow_dispatch`
# This trigger can be found at https://github.com/argumentcomputer/zk-light-clients/actions/workflows/bench.yml
#
# The benchmark report can be found in the logs and as a comment on the latest commit on `dev`.
# The report can also be sent as a Zulip message to https://zulip.argument.xyz
name: Light client benchmark
on:
  workflow_dispatch:
    inputs:
      # Which light client to bench, e.g. `aptos`, `ethereum` or `kadena`
      light-client:
        description: 'Name of the light client to benchmark'
        type: string
        required: true
      # Name of the `light-client` benchmark to run, e.g. `inclusion`, `epoch_change` (for Aptos), `committee_change` (for Ethereum)
      # Runs in the `light-client` directory, so it cannot benchmark `proof_server` or `programs`
      bench-name:
        description: 'Name of the benchmark to run'
        type: string
        required: true
      # List of comma-separated env vars, e.g. `RUST_LOG=debug,SNARK=1`
      # `RUSTFLAGS="-C target-cpu=native --cfg tokio_unstable -C opt-level=3"` is set by default
      env:
        description: 'List of comma-separated environment variables'
        type: string
        required: false
      # Optionally send a message to the below Zulip streams
      # Defaults to false
      zulip:
        description: 'Send the report to Zulip'
        type: boolean
        required: false
      # User(s) to whom to send a private DM (optional)
      # Comma-separated list of user ID integers, e.g. `11,12` (IDs can be found in user profiles)
      # If not specified, sends to a stream/topic pair instead
      private:
        description: 'Send DM to given user ID(s)'
        type: string
        required: false
      # Zulip stream in which to send the message (optional)
      # Ignored if `private` input is specified
      # Defaults to `light_client` stream
      channel:
        description: 'Send message to channel (default is `light-client`). Ignored if DM'
        type: string
        required: false
      # Zulip topic in which to send the message (optional)
      # Ignored if `private` input is specified
      # Defaults to `chat`
      topic:
        description: 'Send message to topic (default is `chat`). Ignored if DM'
        type: string
        required: false
  schedule:
    # Bench report on Tuesdays at 11pm EST/12pm EDT
    - cron: '0 16 * * 4'

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  benchmark:
    name: Light client benchmark
    runs-on: warp-custom-r7iz-metal-16xl
    steps:
      - uses: actions/checkout@v4
        with:
          repository: argumentcomputer/ci-workflows
      - uses: ./.github/actions/ci-env
      - uses: actions/checkout@v4
      - name: Setup CI
        uses: ./.github/actions/setup
      - name: Install extra deps
        run: |
          sudo apt-get update && sudo apt-get install -y python3-pip
          pip3 --version
          pip3 install --break-system-packages jtbl
          echo 'PATH="/$HOME/.local/bin:$PATH"' >> ~/.profile
          source ~/.profile
          jtbl --version
      - name: Set env
        run: |
          # Default benchmark settings optimized for light clients, can be overwritten with `env` input
          echo "RUSTFLAGS='-C target-cpu=native --cfg tokio_unstable -C opt-level=3'" | tee -a $GITHUB_ENV
          echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV
          echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV
          echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV
          echo "SHARD_CHUNKING_MULTIPLIER=256" | tee -a $GITHUB_ENV
          echo "SNARK=1" | tee -a $GITHUB_ENV

          IFS=',' read -ra ENV_VARS <<< "${{ inputs.env }}"
          for VAR in "${ENV_VARS[@]}"; do
            VAR_NAME="${VAR%%=*}"
            VAR_VALUE="${VAR#*=}"
            echo "${VAR_NAME}=${VAR_VALUE}" | tee -a $GITHUB_ENV
          done
      - name: Parse Zulip inputs
        run: |
          if [[ "${{ inputs.zulip }}" == "true" ]]; then
            if [[ ! -z "${{ inputs.private }}" ]]; then
              TYPE="private"
              # Stream = private DM
              STREAM="${{ inputs.private }}"
            else
              TYPE="stream"
              if [[ ! -z "${{ inputs.stream }}" ]]; then
                STREAM="${{ inputs.stream }}"
              elif [[ -z "$STREAM" ]]; then
                STREAM="light_client"
              fi
              if [[ ! -z "${{ inputs.topic }}" ]]; then
                TOPIC="${{ inputs.topic }}"
              elif [[ -z "$TOPIC" ]]; then
                TOPIC="chat"
              fi
            fi
            echo "TYPE=$TYPE" | tee -a $GITHUB_ENV
            echo "STREAM=$STREAM" | tee -a $GITHUB_ENV
            echo "TOPIC=$TOPIC" | tee -a $GITHUB_ENV
          fi
      - name: Run benchmarks
        id: run-benchmarks
        run: |
          make bench-ci BENCH=${{ inputs.bench-name }} 2>&1 | tee out.txt

          grep 'cycles=' out.txt > cycles.txt
          grep 'proving_time' out.txt > timings.txt

          while IFS=$'\t' read -r f1 f2
          do
            num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }')
            echo "$f2" | jq -c ". += {\"cycles\": $num_cycles}" >> summary.json
          done < <(paste cycles.txt timings.txt)

          echo "# Benchmark Results " | tee -a summary.md
          echo "## ${{ inputs.bench-name }} Prove" | tee -a summary.md
          cat summary.json | jtbl -m | tee -a summary.md
          echo "" | tee -a summary.md

          echo "Time unit = milliseconds" | tee -a summary.md
          echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md

          echo "report<<EOF" >> $GITHUB_OUTPUT
          cat summary.md >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT
        working-directory: ${{ github.workspace }}/${{ inputs.package }}/light-client
      - name: Write bench on commit comment
        uses: peter-evans/commit-comment@v3
        with:
          body-path: ${{ github.workspace }}/${{ inputs.package }}/light-client/summary.md
      - name: Send report to Zulip
        if: inputs.zulip
        uses: zulip/github-actions-zulip/send-message@v1
        with:
          api-key: ${{ secrets.ZULIP_API_KEY }}
          email: "aineko-bot@zulip.lurk-lab.com"
          organization-url: "https://zulip.lurk-lab.com"
          to: "${{ env.STREAM }}"
          type: "${{ env.TYPE }}"
          # Ignored if `type: private`
          topic: "${{ env.TOPIC }}"
          content: "${{ steps.run-benchmarks.outputs.report }}"