feat: multi file context #232
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| testbed: | |
| strategy: | |
| matrix: | |
| repo: | |
| - { name: simple, key: simple, parallel: 8, node: false, python: false } | |
| - { name: mmaitre314/picklescan, key: picklescan, parallel: 8, node: false, python: true } | |
| - { name: huggingface/huggingface_hub, key: huggingface_hub, parallel: 8, node: false, python: true } | |
| - { name: tiangolo/fastapi, key: fastapi, parallel: 8, node: false, python: true } | |
| - { name: encode/starlette, key: starlette, parallel: 8, node: false, python: true } | |
| - { name: lancedb/lancedb, key: lancedb, parallel: 2, node: false, python: false } | |
| # - { name: lancedb/lance, key: lance, parallel: 2, node: false, python: false } | |
| - { name: tkaitchuck/constrandom, key: constrandom, parallel: 8, node: false, python: false } | |
| - { name: jaemk/cached, key: cached, parallel: 4, node: false, python: false } | |
| - { name: smol-rs/async-executor, key: async-executor, parallel: 4, node: false, python: false } | |
| - { name: gcanti/io-ts, key: io-ts, parallel: 8, node: true, python: false } | |
| - { name: colinhacks/zod, key: zod, parallel: 8, node: true, python: false } | |
| - { name: helix-editor/helix, key: helix, parallel: 2, node: false, python: false } | |
| runs-on: [self-hosted, intel-cpu, 8-cpu, ci] | |
| container: | |
| image: ubuntu:22.04 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| run: | | |
| apt update | |
| DEBIAN_FRONTEND=noninteractive apt install -y pkg-config protobuf-compiler libssl-dev curl build-essential git-all gfortran | |
| - name: Install Rust toolchain | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| rustflags: '' | |
| toolchain: nightly | |
| - name: Install Python 3.10 | |
| uses: actions/setup-python@v5 | |
| if: matrix.repo.python | |
| with: | |
| python-version: '3.10' | |
| - name: Install node 18 | |
| uses: actions/setup-node@v4 | |
| if: matrix.repo.node | |
| with: | |
| node-version: 18 | |
| - name: Install yarn | |
| if: matrix.repo.node | |
| run: | | |
| npm i -g yarn | |
| - name: Set up cargo cache | |
| uses: actions/cache@v4 | |
| continue-on-error: false | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} | |
| - name: Build project | |
| run: cargo build -r | |
| - name: Run testbed | |
| run: 'cargo run --bin testbed -r -- --api-token $API_TOKEN -r `pwd`/crates/testbed/repositories-ci.yaml -f ${{ matrix.repo.name }} -p ${{ matrix.repo.parallel }}' | |
| if: github.event_name == 'push' || github.event_name == 'pull_request' | |
| env: | |
| API_TOKEN: ${{ secrets.API_TOKEN }} | |
| LOG_LEVEL: ${{ github.run_attempt > 1 && 'debug' || 'info' }} | |
| - name: Run testbed | |
| run: 'cargo run --bin testbed -r -- --api-token $API_TOKEN -f ${{ matrix.repo.name }} -p ${{ matrix.repo.parallel }}' | |
| if: github.event_name == 'workflow_dispatch' | |
| env: | |
| API_TOKEN: ${{ secrets.API_TOKEN }} | |
| LOG_LEVEL: ${{ github.run_attempt > 1 && 'debug' || 'info' }} | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: results-${{ matrix.repo.key }} | |
| path: ./results.json | |
| comment_results: | |
| needs: [testbed] | |
| runs-on: [self-hosted, intel-cpu, 8-cpu, ci] | |
| container: | |
| image: ubuntu:22.04 | |
| steps: | |
| - name: Install dependencies | |
| run: | | |
| apt update | |
| apt install -y jq | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: results-simple | |
| path: results-simple | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: results-picklescan | |
| path: results-picklescan | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: results-huggingface_hub | |
| path: results-huggingface_hub | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: results-fastapi | |
| path: results-fastapi | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: results-starlette | |
| path: results-starlette | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: results-lancedb | |
| path: results-lancedb | |
| # - uses: actions/download-artifact@v4 | |
| # with: | |
| # name: results-lance | |
| # path: results-lance | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: results-constrandom | |
| path: results-constrandom | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: results-cached | |
| path: results-cached | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: results-async-executor | |
| path: results-async-executor | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: results-io-ts | |
| path: results-io-ts | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: results-zod | |
| path: results-zod | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: results-helix | |
| path: results-helix | |
| - name: Display structure of downloaded files | |
| run: ls -R | |
| - name: output to markdown | |
| run: | | |
| cat > results.md <<EOF | |
| | Repository name | Source type | Average hole completion time (s) | Pass percentage | | |
| | :-------------- | :---------- | -------------------------------: | --------------: | | |
| EOF | |
| cat **/results.json | jq -r '"| \(.[0].repo_name) | \(.[0].source_type) | \(.[0].avg_hole_completion_time_ms) | \(.[0].pass_percentage)% |"' >> results.md | |
| cat >> results.md <<EOF | |
| **Note:** The "hole completion time" represents the full process of: | |
| - copying files from the setup cache directory | |
| - replacing the code from the file with a completion from the model | |
| - building the project | |
| - running the tests | |
| EOF | |
| - name: Find Comment | |
| uses: peter-evans/find-comment@v2 | |
| id: fc | |
| if: github.event_name == 'pull_request' | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| comment-author: 'github-actions[bot]' | |
| body-includes: '| Repository name | Source type | Average hole completion time (s) | Pass percentage |' | |
| - name: Create or update comment | |
| if: github.event_name == 'pull_request' | |
| uses: peter-evans/create-or-update-comment@v3 | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body-path: results.md | |
| edit-mode: replace | |