-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Idea I got at work today so wanted to quickly implement. Benchmarks the example using [oha](https://github.com/hatoo/oha/) and then comments the result on every PR. Have ideas for performance improvements in future so will be good to know if my efforts actually do anything. Example echoed out a lot so numbers will be off on first run. Don't know how consistent it will be so might need to add better stats (student t test?)
- Loading branch information
1 parent
9958a0b
commit 817ab36
Showing
6 changed files
with
110 additions
and
52 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
#!/usr/bin/bash | ||
# Be on correct branch | ||
git checkout $1 | ||
|
||
nimble install | ||
# Compile the example | ||
nim c -f -d:release example.nim | ||
# Start it | ||
./example & | ||
# And then generate the output | ||
oha --no-tui -z 30sec -j http://127.0.0.1:8080/person/foo/9 > "${1}.json" | ||
|
||
# And stop the example server | ||
pkill -P $$ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
name: Benchmarking | ||
|
||
on: | ||
pull_request: | ||
|
||
permissions: | ||
pull-requests: write | ||
|
||
jobs: | ||
bench: | ||
runs-on: ubuntu-latest | ||
name: Benchmark performance | ||
steps: | ||
- name: Setup Nim Enviroment | ||
uses: actions/checkout@v3 | ||
with: | ||
fetch-depth: 0 | ||
|
||
- uses: jiro4989/setup-nim-action@v2 | ||
with: | ||
use-nightlies: true | ||
repo-token: ${{ secrets.GITHUB_TOKEN }} | ||
nim-version: stable | ||
|
||
- name: Install oha | ||
run: | | ||
echo "deb [signed-by=/usr/share/keyrings/azlux-archive-keyring.gpg] http://packages.azlux.fr/debian/ stable main" | sudo tee /etc/apt/sources.list.d/azlux.list | ||
sudo wget -O /usr/share/keyrings/azlux-archive-keyring.gpg https://azlux.fr/repo.gpg | ||
sudo apt update | ||
sudo apt install oha | ||
- name: Copy script | ||
run: cp .github/workflows/bench.sh bench.sh | ||
|
||
- name: Run on master | ||
run: ./bench.sh master | ||
|
||
- name: Run on branch | ||
run: ./bench.sh $GITHUB_SHA | ||
|
||
- name: Compare | ||
run: | | ||
nim r .github/workflows/stats.nim master.json ${GITHUB_SHA}.json > comment.md | ||
cat comment.md | ||
- name: Comment results | ||
uses: thollander/actions-comment-pull-request@v3 | ||
with: | ||
file-path: comment.md | ||
comment-tag: execution |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
## Reads in oha output and creates a markdown table of stats | ||
|
||
import std/[json, os, strutils] | ||
|
||
type | ||
Summary = object | ||
successRate, total, slowest, fastest, average, requestsPerSec: float | ||
Stat = object | ||
summary: Summary | ||
|
||
let | ||
baseline = parseFile(paramStr(1)).to(Stat).summary | ||
candidate = parseFile(paramStr(2)).to(Stat).summary | ||
|
||
template formatVal(x: float): string = formatFloat(x, ffDecimal, 5) | ||
|
||
template writeRow(metric: untyped, name: string, lowerBetter = true) = | ||
let | ||
diff = (when lowerBetter: -1 else: 1) * (candidate.metric - baseline.metric) | ||
percentage = (diff / baseline.metric) * 100 | ||
sign = (if percentage >= 0: "+" else: "") | ||
formattedPercentage = sign & formatFloat(percentage, ffDecimal, 2) | ||
echo "|", name, "|", formatVal(baseline.metric), "|", formatVal(candidate.metric), "|", formattedPercentage, "|" | ||
|
||
echo "| Metric | Baseline | Candidate | Difference (%) |" | ||
echo "|--------|----------|-----------|----------------|" | ||
writeRow(successRate, "Success Rate") | ||
writeRow(slowest, "Slowest") | ||
writeRow(fastest, "Fastest") | ||
writeRow(average, "Average") | ||
writeRow(requestsPerSec, "Req/s", false) |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters