Skip to content

Commit

Permalink
Performance testing in the CI (#54)
Browse files Browse the repository at this point in the history
Idea I got at work today so wanted to quickly implement. Benchmarks the
example using [oha](https://github.com/hatoo/oha/) and then comments the
result on every PR. Have ideas for performance improvements in future so
will be good to know if my efforts actually do anything. Example echoed
out a lot so numbers will be off on first run. Don't know how consistent
it will be so might need to add better stats (student t test?)
  • Loading branch information
ire4ever1190 authored Oct 29, 2024
1 parent 9958a0b commit 817ab36
Show file tree
Hide file tree
Showing 6 changed files with 110 additions and 52 deletions.
14 changes: 14 additions & 0 deletions .github/workflows/bench.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/usr/bin/bash
# Be on correct branch
git checkout $1

nimble install
# Compile the example
nim c -f -d:release example.nim
# Start it
./example &
# And then generate the output
oha --no-tui -z 30sec -j http://127.0.0.1:8080/person/foo/9 > "${1}.json"

# And stop the example server
pkill -P $$
50 changes: 50 additions & 0 deletions .github/workflows/performance.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
name: Benchmarking

on:
pull_request:

permissions:
pull-requests: write

jobs:
bench:
runs-on: ubuntu-latest
name: Benchmark performance
steps:
- name: Setup Nim Enviroment
uses: actions/checkout@v3
with:
fetch-depth: 0

- uses: jiro4989/setup-nim-action@v2
with:
use-nightlies: true
repo-token: ${{ secrets.GITHUB_TOKEN }}
nim-version: stable

- name: Install oha
run: |
echo "deb [signed-by=/usr/share/keyrings/azlux-archive-keyring.gpg] http://packages.azlux.fr/debian/ stable main" | sudo tee /etc/apt/sources.list.d/azlux.list
sudo wget -O /usr/share/keyrings/azlux-archive-keyring.gpg https://azlux.fr/repo.gpg
sudo apt update
sudo apt install oha
- name: Copy script
run: cp .github/workflows/bench.sh bench.sh

- name: Run on master
run: ./bench.sh master

- name: Run on branch
run: ./bench.sh $GITHUB_SHA

- name: Compare
run: |
nim r .github/workflows/stats.nim master.json ${GITHUB_SHA}.json > comment.md
cat comment.md
- name: Comment results
uses: thollander/actions-comment-pull-request@v3
with:
file-path: comment.md
comment-tag: execution
31 changes: 31 additions & 0 deletions .github/workflows/stats.nim
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
## Reads in oha output and creates a markdown table of stats

import std/[json, os, strutils]

type
Summary = object
successRate, total, slowest, fastest, average, requestsPerSec: float
Stat = object
summary: Summary

let
baseline = parseFile(paramStr(1)).to(Stat).summary
candidate = parseFile(paramStr(2)).to(Stat).summary

template formatVal(x: float): string = formatFloat(x, ffDecimal, 5)

template writeRow(metric: untyped, name: string, lowerBetter = true) =
let
diff = (when lowerBetter: -1 else: 1) * (candidate.metric - baseline.metric)
percentage = (diff / baseline.metric) * 100
sign = (if percentage >= 0: "+" else: "")
formattedPercentage = sign & formatFloat(percentage, ffDecimal, 2)
echo "|", name, "|", formatVal(baseline.metric), "|", formatVal(candidate.metric), "|", formattedPercentage, "|"

echo "| Metric | Baseline | Candidate | Difference (%) |"
echo "|--------|----------|-----------|----------------|"
writeRow(successRate, "Success Rate")
writeRow(slowest, "Slowest")
writeRow(fastest, "Fastest")
writeRow(average, "Average")
writeRow(requestsPerSec, "Req/s", false)
32 changes: 0 additions & 32 deletions benchmark.nim

This file was deleted.

31 changes: 15 additions & 16 deletions example.nim
Original file line number Diff line number Diff line change
Expand Up @@ -7,36 +7,35 @@ import strformat
#

"/" -> get:
ctx.send "Mike is running!"
ctx.send "Mike is running!"

"/hello" -> post:
let names = ctx.json(seq[string])
for name in names:
echo name
ctx.send "OK"
let names = ctx.json(seq[string])
for name in names:
echo name
ctx.send "OK"

"/shutdown" -> get:
quit 0
quit 0

#
# Custom data
#

type
Person = ref object of RootObj
name: string
age: int
Person = ref object of RootObj
name: string
age: int

"/person/:name/:age" -> beforeGet:
echo ctx.pathParams
ctx &= Person(
name: ctx.pathParams["name"],
age: ctx.pathParams["age"].parseInt()
)
ctx &= Person(
name: ctx.pathParams["name"],
age: ctx.pathParams["age"].parseInt()
)

"/person/:name/:age" -> get:
let person = ctx[Person]
ctx.send fmt"Hello {person.name} aged {person.age}"
let person = ctx[Person]
ctx.send fmt"Hello {person.name} aged {person.age}"


run()
4 changes: 0 additions & 4 deletions mike.nimble
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,3 @@ requires "httpx >= 0.3.7"

task ex, "Runs the example":
selfExec "c -f -d:debug -r example"

task bench, "Runs a benchmark and saves it to a file with the current time":
for cmd in ["compile", "bench"]:
selfExec "r -d:release --gc:arc --opt:size benchmark " & cmd

0 comments on commit 817ab36

Please sign in to comment.