Test e2e #56
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Runs benchmarks on self-hosted infra via `workflow_dispatch` and scheduled on Thursdays at 2pm UTC | |
| # | |
| # `workflow_dispatch` trigger | |
| # The workflow can be run at https://github.com/argumentcomputer/zk-light-clients/actions/workflows/bench.yml | |
| # The benchmark report can be found in the logs and as a comment on the latest commit on `dev`. | |
| # The report can also be sent as a Zulip message to https://zulip.argument.xyz | |
| # | |
| # `schedule` trigger | |
| # The workflow runs every week on Thursday at 2pm UTC | |
| # It runs all light client benchmarks and sends them to Zulip | |
| name: Light client benchmark | |
| on: | |
| push: | |
| branches: | |
| - "ci-bench" | |
| schedule: | |
| # Full report on Thursday, 2pm UTC | |
| - cron: '0 14 * * 4' | |
| workflow_dispatch: | |
| inputs: | |
| # Which light client to bench, e.g. `aptos`, `ethereum` or `kadena` | |
| light-client: | |
| description: 'Name of the light client to benchmark' | |
| type: string | |
| required: true | |
| # Name of the `light-client` benchmark to run, e.g. | |
| # Aptos: `inclusion` or `epoch_change` | |
| # Ethereum: `inclusion or `committee_change` | |
| # Kadena: `longest_chain` or `spv` | |
| # Runs in the `light-client` directory, so it cannot benchmark `proof_server` or `programs` | |
| bench-name: | |
| description: 'Name of the benchmark to run' | |
| type: string | |
| required: true | |
| # List of comma-separated env vars, e.g. `RUST_LOG=debug,MODE=SNARK` | |
| # `RUSTFLAGS="-C target-cpu=native --cfg tokio_unstable -C opt-level=3"` is set by default | |
| env: | |
| description: 'List of comma-separated environment variables' | |
| type: string | |
| required: false | |
| # Optionally send a message to the below Zulip streams | |
| # Defaults to false | |
| zulip: | |
| description: 'Send the report to Zulip' | |
| type: boolean | |
| required: false | |
| # User(s) to whom to send a private DM (optional) | |
| # Comma-separated list of user ID integers, e.g. `11,12` (IDs can be found in user profiles) | |
| # If not specified, sends to a stream/topic pair instead | |
| private: | |
| description: 'Send DM to given user ID(s)' | |
| type: string | |
| required: false | |
| # Zulip stream in which to send the message (optional) | |
| # Ignored if `private` input is specified | |
| # Defaults to `light_client` stream | |
| channel: | |
| description: 'Send message to channel (default is `light-client`). Ignored if `private` input is specified' | |
| type: string | |
| required: false | |
| # Zulip topic in which to send the message (optional) | |
| # Ignored if `private` input is specified | |
| # Defaults to `chat` | |
| topic: | |
| description: 'Send message to topic (default is `chat`). Ignored if `private` input is specified' | |
| type: string | |
| required: false | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| benchmark-manual: | |
| name: LC bench (manual) | |
| if: github.event_name == 'workflow_dispatch' | |
| runs-on: warp-custom-r7iz-metal-32xl | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: argumentcomputer/ci-workflows | |
| - uses: ./.github/actions/ci-env | |
| - uses: actions/checkout@v4 | |
| - name: Setup CI | |
| uses: ./.github/actions/setup | |
| - name: Set env | |
| run: | | |
| # Default benchmark settings optimized for light clients, can be overwritten with `env` input | |
| echo "RUSTFLAGS=-C target-cpu=native --cfg tokio_unstable -C opt-level=3" | tee -a $GITHUB_ENV | |
| echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV | |
| echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV | |
| echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV | |
| echo "SHARD_CHUNKING_MULTIPLIER=1" | tee -a $GITHUB_ENV | |
| echo "MODE=SNARK" | tee -a $GITHUB_ENV | |
| IFS=',' read -ra ENV_VARS <<< "${{ inputs.env }}" | |
| for VAR in "${ENV_VARS[@]}"; do | |
| VAR_NAME="${VAR%%=*}" | |
| VAR_VALUE="${VAR#*=}" | |
| echo "${VAR_NAME}=${VAR_VALUE}" | tee -a $GITHUB_ENV | |
| done | |
| - name: Parse Zulip inputs | |
| run: | | |
| if [[ "${{ inputs.zulip }}" == "true" ]]; then | |
| if [[ ! -z "${{ inputs.private }}" ]]; then | |
| TYPE="private" | |
| # Stream = private DM | |
| STREAM="${{ inputs.private }}" | |
| else | |
| TYPE="stream" | |
| if [[ ! -z "${{ inputs.stream }}" ]]; then | |
| STREAM="${{ inputs.stream }}" | |
| elif [[ -z "$STREAM" ]]; then | |
| STREAM="light_client" | |
| fi | |
| if [[ ! -z "${{ inputs.topic }}" ]]; then | |
| TOPIC="${{ inputs.topic }}" | |
| elif [[ -z "$TOPIC" ]]; then | |
| TOPIC="chat" | |
| fi | |
| fi | |
| echo "TYPE=$TYPE" | tee -a $GITHUB_ENV | |
| echo "STREAM=$STREAM" | tee -a $GITHUB_ENV | |
| echo "TOPIC=$TOPIC" | tee -a $GITHUB_ENV | |
| fi | |
| - name: Run benchmark and create report | |
| id: run-benchmark | |
| uses: ./.github/actions/bench | |
| with: | |
| light-client: ${{ inputs.light-client }} | |
| benchmark: ${{ inputs.bench-name }} | |
| - name: Send report to Zulip | |
| if: inputs.zulip | |
| uses: zulip/github-actions-zulip/send-message@v1 | |
| with: | |
| api-key: ${{ secrets.ZULIP_API_KEY }} | |
| email: "[email protected]" | |
| organization-url: "https://zulip.argument.xyz" | |
| to: "${{ env.STREAM }}" | |
| type: "${{ env.TYPE }}" | |
| # Ignored if `type: private` | |
| topic: "${{ env.TOPIC }}" | |
| content: "${{ steps.run-benchmark.outputs.report }}" | |
| benchmark-scheduled: | |
| name: LC bench (scheduled) | |
| if: github.event_name == 'schedule' | |
| runs-on: warp-custom-r7iz-metal-32xl | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - light-client: aptos | |
| benchmark: inclusion | |
| - light-client: aptos | |
| benchmark: epoch_change | |
| - light-client: ethereum | |
| benchmark: inclusion | |
| - light-client: ethereum | |
| benchmark: committee_change | |
| - light-client: kadena | |
| benchmark: spv | |
| - light-client: kadena | |
| benchmark: longest_chain | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: argumentcomputer/ci-workflows | |
| - uses: ./.github/actions/ci-env | |
| - uses: actions/checkout@v4 | |
| - name: Setup CI | |
| uses: ./.github/actions/setup | |
| - name: Set env | |
| run: | | |
| # Default benchmark settings optimized for light clients, can be overwritten with `env` input | |
| echo "RUSTFLAGS=-C target-cpu=native --cfg tokio_unstable -C opt-level=3" | tee -a $GITHUB_ENV | |
| echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV | |
| echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV | |
| echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV | |
| echo "SHARD_CHUNKING_MULTIPLIER=1" | tee -a $GITHUB_ENV | |
| echo "MODE=SNARK" | tee -a $GITHUB_ENV | |
| - name: Set Zulip env | |
| run: | | |
| echo "TYPE=stream" | tee -a $GITHUB_ENV | |
| echo "STREAM=light-client" | tee -a $GITHUB_ENV | |
| echo "TOPIC=Benchmark Reports" | tee -a $GITHUB_ENV | |
| - name: Run benchmark and create report | |
| id: run-benchmark | |
| uses: ./.github/actions/bench | |
| with: | |
| light-client: ${{ matrix.light-client }} | |
| benchmark: ${{ matrix.benchmark }} | |
| - name: Send report to Zulip | |
| uses: zulip/github-actions-zulip/send-message@v1 | |
| with: | |
| api-key: ${{ secrets.ZULIP_API_KEY }} | |
| email: "[email protected]" | |
| organization-url: "https://zulip.argument.xyz" | |
| to: "${{ env.STREAM }}" | |
| type: "${{ env.TYPE }}" | |
| # Ignored if `type: private` | |
| topic: "${{ env.TOPIC }}" | |
| content: "${{ steps.run-benchmark.outputs.report }}" | |
| benchmark-test: | |
| name: LC bench (test) | |
| runs-on: warp-custom-r7iz-metal-32xl | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - light-client: aptos | |
| benchmark: inclusion | |
| - light-client: aptos | |
| benchmark: epoch_change | |
| - light-client: ethereum | |
| benchmark: inclusion | |
| - light-client: ethereum | |
| benchmark: committee_change | |
| - light-client: kadena | |
| benchmark: spv | |
| - light-client: kadena | |
| benchmark: longest_chain | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: argumentcomputer/ci-workflows | |
| - uses: ./.github/actions/ci-env | |
| - uses: actions/checkout@v4 | |
| - name: Setup CI | |
| uses: ./.github/actions/setup | |
| - name: Set env | |
| run: | | |
| # Default benchmark settings optimized for light clients, can be overwritten with `env` input | |
| echo "RUSTFLAGS=-C target-cpu=native --cfg tokio_unstable -C opt-level=3" | tee -a $GITHUB_ENV | |
| echo "SHARD_SIZE=4194304" | tee -a $GITHUB_ENV | |
| echo "SHARD_BATCH_SIZE=0" | tee -a $GITHUB_ENV | |
| echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV | |
| echo "SHARD_CHUNKING_MULTIPLIER=1" | tee -a $GITHUB_ENV | |
| echo "MODE=SNARK" | tee -a $GITHUB_ENV | |
| - name: Set Zulip env | |
| run: | | |
| echo "TYPE=private" | tee -a $GITHUB_ENV | |
| echo "STREAM=11" | tee -a $GITHUB_ENV | |
| echo "TOPIC=" | tee -a $GITHUB_ENV | |
| - name: Run benchmark and create report | |
| id: run-benchmark | |
| uses: ./.github/actions/bench | |
| with: | |
| light-client: ${{ matrix.light-client }} | |
| benchmark: ${{ matrix.benchmark }} | |
| - name: Send report to Zulip | |
| uses: zulip/github-actions-zulip/send-message@v1 | |
| with: | |
| api-key: ${{ secrets.ZULIP_API_KEY }} | |
| email: "[email protected]" | |
| organization-url: "https://zulip.argument.xyz" | |
| to: "${{ env.STREAM }}" | |
| type: "${{ env.TYPE }}" | |
| # Ignored if `type: private` | |
| topic: "${{ env.TOPIC }}" | |
| content: "${{ steps.run-benchmark.outputs.report }}" |