Add scheduled workflow #41
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Runs benchmarks on self-hosted infra via `workflow_dispatch` and scheduled on Mondays at 9am EST | |
| # | |
| # `workflow_dispatch` trigger | |
| # The workflow can be run at https://github.com/argumentcomputer/zk-light-clients/actions/workflows/bench.yml | |
| # The benchmark report can be found in the logs and as a comment on the latest commit on `dev`. | |
| # The report can also be sent as a Zulip message to https://zulip.argument.xyz | |
| # | |
| # `schedule` trigger | |
| # The workflow runs every week on Monday at 9am EST | |
| # It runs all light client benchmarks and sends them to Zulip | |
| name: Light client benchmark | |
| on: | |
| schedule: | |
| # Full report on Mondays at 9am EST/10am EDT | |
| - cron: '0 14 * * 1' | |
| workflow_dispatch: | |
| inputs: | |
| # Which light client to bench, e.g. `aptos`, `ethereum` or `kadena` | |
| light-client: | |
| description: 'Name of the light client to benchmark' | |
| type: string | |
| required: true | |
| # Name of the `light-client` benchmark to run, e.g. | |
| # Aptos: `inclusion` or `epoch_change` | |
| # Ethereum: `inclusion or `committee_change` | |
| # Kadena: `longest_chain` or `spv` | |
| # Runs in the `light-client` directory, so it cannot benchmark `proof_server` or `programs` | |
| bench-name: | |
| description: 'Name of the benchmark to run' | |
| type: string | |
| required: true | |
| # List of comma-separated env vars, e.g. `RUST_LOG=debug,SNARK=1` | |
| # `RUSTFLAGS="-C target-cpu=native --cfg tokio_unstable -C opt-level=3"` is set by default | |
| env: | |
| description: 'List of comma-separated environment variables' | |
| type: string | |
| required: false | |
| # Optionally send a message to the below Zulip streams | |
| # Defaults to false | |
| zulip: | |
| description: 'Send the report to Zulip' | |
| type: boolean | |
| required: false | |
| # User(s) to whom to send a private DM (optional) | |
| # Comma-separated list of user ID integers, e.g. `11,12` (IDs can be found in user profiles) | |
| # If not specified, sends to a stream/topic pair instead | |
| private: | |
| description: 'Send DM to given user ID(s)' | |
| type: string | |
| required: false | |
| # Zulip stream in which to send the message (optional) | |
| # Ignored if `private` input is specified | |
| # Defaults to `light_client` stream | |
| channel: | |
| description: 'Send message to channel (default is `light-client`). Ignored if `private` input is specified' | |
| type: string | |
| required: false | |
| # Zulip topic in which to send the message (optional) | |
| # Ignored if `private` input is specified | |
| # Defaults to `chat` | |
| topic: | |
| description: 'Send message to topic (default is `chat`). Ignored if `private` input is specified' | |
| type: string | |
| required: false | |
| schedule: | |
| # Bench report on Tuesdays at 11pm EST/12pm EDT | |
| - cron: '0 16 * * 4' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| benchmark-manual: | |
| name: Light client benchmark (manual) | |
| if: github.event_name == 'workflow_dispatch' | |
| runs-on: warp-custom-r7iz-metal-32xl | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: argumentcomputer/ci-workflows | |
| - uses: ./.github/actions/ci-env | |
| - uses: actions/checkout@v4 | |
| - name: Setup CI | |
| uses: ./.github/actions/setup | |
| - name: Install extra deps | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y python3-pip | |
| pip3 --version | |
| pip3 install --break-system-packages jtbl | |
| echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.profile | |
| source ~/.profile | |
| which jtbl | |
| - name: Set env | |
| run: | | |
| # Default benchmark settings optimized for light clients, can be overwritten with `env` input | |
| echo "RUSTFLAGS=-C target-cpu=native --cfg tokio_unstable -C opt-level=3" | tee -a $GITHUB_ENV | |
| echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV | |
| echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV | |
| echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV | |
| echo "SHARD_CHUNKING_MULTIPLIER=256" | tee -a $GITHUB_ENV | |
| echo "SNARK=1" | tee -a $GITHUB_ENV | |
| IFS=',' read -ra ENV_VARS <<< "${{ inputs.env }}" | |
| for VAR in "${ENV_VARS[@]}"; do | |
| VAR_NAME="${VAR%%=*}" | |
| VAR_VALUE="${VAR#*=}" | |
| echo "${VAR_NAME}=${VAR_VALUE}" | tee -a $GITHUB_ENV | |
| done | |
| - name: Parse Zulip inputs | |
| run: | | |
| if [[ "${{ inputs.zulip }}" == "true" ]]; then | |
| if [[ ! -z "${{ inputs.private }}" ]]; then | |
| TYPE="private" | |
| # Stream = private DM | |
| STREAM="${{ inputs.private }}" | |
| else | |
| TYPE="stream" | |
| if [[ ! -z "${{ inputs.stream }}" ]]; then | |
| STREAM="${{ inputs.stream }}" | |
| elif [[ -z "$STREAM" ]]; then | |
| STREAM="light_client" | |
| fi | |
| if [[ ! -z "${{ inputs.topic }}" ]]; then | |
| TOPIC="${{ inputs.topic }}" | |
| elif [[ -z "$TOPIC" ]]; then | |
| TOPIC="chat" | |
| fi | |
| fi | |
| echo "TYPE=$TYPE" | tee -a $GITHUB_ENV | |
| echo "STREAM=$STREAM" | tee -a $GITHUB_ENV | |
| echo "TOPIC=$TOPIC" | tee -a $GITHUB_ENV | |
| fi | |
| - name: Run benchmarks | |
| id: run-benchmarks | |
| run: | | |
| make bench-ci BENCH=${{ inputs.bench-name }} 2>&1 | tee out.txt | |
| grep 'cycles=' out.txt > cycles.txt | |
| grep 'proving_time' out.txt > timings.txt | |
| while IFS=$'\t' read -r f1 f2 | |
| do | |
| num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }') | |
| timings=$(echo "$f2" | jq ' | |
| to_entries | | |
| map( | |
| if .key == "proving_time" or .key == "verifying_time" then | |
| {key, value: (.value / 1000 | floor as $s | "\(($s / 60 | floor) | tostring)min\(($s % 60) | tostring)s")} | |
| else | |
| . | |
| end | |
| ) | | |
| from_entries | |
| ') | |
| echo "$timings" | jq -c --argjson cycles "$num_cycles" '. += {cycles: $cycles}' >> summary.json | |
| done < <(paste cycles.txt timings.txt) | |
| echo '# `${{ inputs.light-client }}` Benchmark Results' | tee -a summary.md | |
| echo '## `${{ inputs.bench-name }}` Prove' | tee -a summary.md | |
| cat summary.json | jtbl -m | tee -a summary.md | |
| echo "" | tee -a summary.md | |
| echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md | |
| echo "report<<EOF" >> $GITHUB_OUTPUT | |
| cat summary.md >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| working-directory: ${{ github.workspace }}/${{ inputs.light-client }}/light-client | |
| - name: Write bench on commit comment | |
| uses: peter-evans/commit-comment@v3 | |
| with: | |
| body-path: ${{ github.workspace }}/${{ inputs.light-client }}/light-client/summary.md | |
| - name: Send report to Zulip | |
| if: inputs.zulip | |
| uses: zulip/github-actions-zulip/send-message@v1 | |
| with: | |
| api-key: ${{ secrets.ZULIP_API_KEY }} | |
| email: "[email protected]" | |
| organization-url: "https://zulip.argument.xyz" | |
| to: "${{ env.STREAM }}" | |
| type: "${{ env.TYPE }}" | |
| # Ignored if `type: private` | |
| topic: "${{ env.TOPIC }}" | |
| content: "${{ steps.run-benchmarks.outputs.report }}" | |
| benchmark-scheduled: | |
| name: Light client bench (scheduled) | |
| if: github.event_name == 'schedule' | |
| runs-on: warp-custom-r7iz-metal-32xl | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| light-client: [aptos, ethereum, kadena] | |
| include: | |
| - benchmark: inclusion | |
| light-client: aptos | |
| - benchmark: epoch_change | |
| light-client: aptos | |
| - benchmark: inclusion | |
| light-client: ethereum | |
| - benchmark: committee_change | |
| light-client: ethereum | |
| - benchmark: spv | |
| light-client: kadena | |
| - benchmark: longest_chain | |
| light-client: kadena | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: argumentcomputer/ci-workflows | |
| - uses: ./.github/actions/ci-env | |
| - uses: actions/checkout@v4 | |
| - name: Setup CI | |
| uses: ./.github/actions/setup | |
| - name: Install extra deps | |
| run: | | |
| sudo apt-get update && sudo apt-get install -y python3-pip | |
| pip3 --version | |
| pip3 install --break-system-packages jtbl | |
| echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.profile | |
| source ~/.profile | |
| which jtbl | |
| - name: Set env | |
| run: | | |
| # Default benchmark settings optimized for light clients, can be overwritten with `env` input | |
| echo "RUSTFLAGS=-C target-cpu=native --cfg tokio_unstable -C opt-level=3" | tee -a $GITHUB_ENV | |
| echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV | |
| echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV | |
| echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV | |
| echo "SHARD_CHUNKING_MULTIPLIER=256" | tee -a $GITHUB_ENV | |
| echo "SNARK=1" | tee -a $GITHUB_ENV | |
| IFS=',' read -ra ENV_VARS <<< "ethereum" | |
| for VAR in "${ENV_VARS[@]}"; do | |
| VAR_NAME="${VAR%%=*}" | |
| VAR_VALUE="${VAR#*=}" | |
| echo "${VAR_NAME}=${VAR_VALUE}" | tee -a $GITHUB_ENV | |
| done | |
| - name: Set env | |
| run: | | |
| echo "TYPE=stream" | tee -a $GITHUB_ENV | |
| echo "STREAM=light-client" | tee -a $GITHUB_ENV | |
| echo "TOPIC=Benchmark Reports" | tee -a $GITHUB_ENV | |
| - name: Run benchmarks | |
| id: run-benchmarks | |
| run: | | |
| make bench-ci BENCH=${{ matrix.benchmark }} 2>&1 | tee out.txt | |
| grep 'cycles=' out.txt > cycles.txt | |
| grep 'proving_time' out.txt > timings.txt | |
| while IFS=$'\t' read -r f1 f2 | |
| do | |
| num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }') | |
| timings=$(echo "$f2" | jq ' | |
| to_entries | | |
| map( | |
| if .key == "proving_time" or .key == "verifying_time" then | |
| {key, value: (.value / 1000 | floor as $s | "\(($s / 60 | floor) | tostring)min\(($s % 60) | tostring)s")} | |
| else | |
| . | |
| end | |
| ) | | |
| from_entries | |
| ') | |
| echo "$timings" | jq -c --argjson cycles "$num_cycles" '. += {cycles: $cycles}' >> summary.json | |
| done < <(paste cycles.txt timings.txt) | |
| echo '# `${{ matrix.light-client }}` Benchmark Results' | tee -a summary.md | |
| echo '## `${{ matrix.benchmark }}` Prove' | tee -a summary.md | |
| cat summary.json | jtbl -m | tee -a summary.md | |
| echo "" | tee -a summary.md | |
| echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md | |
| echo "report<<EOF" >> $GITHUB_OUTPUT | |
| cat summary.md >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| working-directory: ${{ github.workspace }}/${{ matrix.light-client }}/light-client | |
| - name: Write bench on commit comment | |
| uses: peter-evans/commit-comment@v3 | |
| with: | |
| body-path: ${{ github.workspace }}/${{ matrix.light-client }}/light-client/summary.md | |
| - name: Send report to Zulip | |
| uses: zulip/github-actions-zulip/send-message@v1 | |
| with: | |
| api-key: ${{ secrets.ZULIP_API_KEY }} | |
| email: "[email protected]" | |
| organization-url: "https://zulip.argument.xyz" | |
| to: "${{ env.STREAM }}" | |
| type: "${{ env.TYPE }}" | |
| # Ignored if `type: private` | |
| topic: "${{ env.TOPIC }}" | |
| content: "${{ steps.run-benchmarks.outputs.report }}" |