diff --git a/.github/workflows/bench.sh b/.github/workflows/bench.sh new file mode 100755 index 0000000..2e43831 --- /dev/null +++ b/.github/workflows/bench.sh @@ -0,0 +1,14 @@ +#!/usr/bin/bash +# Be on correct branch +git checkout $1 + +nimble install +# Compile the example +nim c -f -d:release example.nim +# Start it +./example & +# And then generate the output +oha --no-tui -z 30sec -j http://127.0.0.1:8080/person/foo/9 > "${1}.json" + +# And stop the example server +pkill -P $$ diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 0000000..3b50f33 --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,50 @@ +name: Benchmarking + +on: + pull_request: + +permissions: + pull-requests: write + +jobs: + bench: + runs-on: ubuntu-latest + name: Benchmark performance + steps: + - name: Setup Nim Enviroment + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: jiro4989/setup-nim-action@v2 + with: + use-nightlies: true + repo-token: ${{ secrets.GITHUB_TOKEN }} + nim-version: stable + + - name: Install oha + run: | + echo "deb [signed-by=/usr/share/keyrings/azlux-archive-keyring.gpg] http://packages.azlux.fr/debian/ stable main" | sudo tee /etc/apt/sources.list.d/azlux.list + sudo wget -O /usr/share/keyrings/azlux-archive-keyring.gpg https://azlux.fr/repo.gpg + sudo apt update + sudo apt install oha + + - name: Copy script + run: cp .github/workflows/bench.sh bench.sh + + - name: Run on master + run: ./bench.sh master + + - name: Run on branch + run: ./bench.sh $GITHUB_SHA + + - name: Compare + run: | + nim r .github/workflows/stats.nim master.json ${GITHUB_SHA}.json > comment.md + cat comment.md + + - name: Comment results + uses: thollander/actions-comment-pull-request@v3 + with: + file-path: comment.md + comment-tag: execution diff --git a/.github/workflows/stats.nim b/.github/workflows/stats.nim new file mode 100644 index 0000000..7ff45b2 --- /dev/null +++ b/.github/workflows/stats.nim @@ -0,0 +1,31 @@ +## Reads in oha output and creates a markdown table of stats + +import std/[json, os, strutils] + +type + Summary = object + successRate, total, slowest, fastest, average, requestsPerSec: float + Stat = object + summary: Summary + +let + baseline = parseFile(paramStr(1)).to(Stat).summary + candidate = parseFile(paramStr(2)).to(Stat).summary + +template formatVal(x: float): string = formatFloat(x, ffDecimal, 5) + +template writeRow(metric: untyped, name: string, lowerBetter = true) = + let + diff = (when lowerBetter: -1 else: 1) * (candidate.metric - baseline.metric) + percentage = (diff / baseline.metric) * 100 + sign = (if percentage >= 0: "+" else: "") + formattedPercentage = sign & formatFloat(percentage, ffDecimal, 2) + echo "|", name, "|", formatVal(baseline.metric), "|", formatVal(candidate.metric), "|", formattedPercentage, "|" + +echo "| Metric | Baseline | Candidate | Difference (%) |" +echo "|--------|----------|-----------|----------------|" +writeRow(successRate, "Success Rate") +writeRow(slowest, "Slowest") +writeRow(fastest, "Fastest") +writeRow(average, "Average") +writeRow(requestsPerSec, "Req/s", false) diff --git a/benchmark.nim b/benchmark.nim deleted file mode 100644 index 837ae47..0000000 --- a/benchmark.nim +++ /dev/null @@ -1,32 +0,0 @@ -# -# Used to run benchmarks to check that code isn't creating slowdowns. -# Also does other utility functions like generating graphs and basic statistics over past runs -# -import std/osproc -import std/os -import std/strformat -import std/times - -const gc = when defined(useOrc): "orc" else: "refc" - -const - buildCmd = fmt"nim c -d:release -d:lto --gc:{gc} -o:benchserver example.nim" # Command to build server - benchCmd = "wrk http://127.0.0.1:8080/" # Run wrk to benchmark server - -case paramStr(1): - of "compile": - # Compile the benchmark server, should only be done once per code change - echo execProcess buildCmd - of "bench": - # Run wrk and saves output to file in benchmark/ with UNIX - # timestamp as it's name - if not fileExists "benchserver": - echo "Compile the bench server first" - quit 1 - let serverProcess = startProcess "./benchserver" - sleep 100 # Give server time to boot - discard existsOrCreateDir("benchmark/") - ("benchmark/" & $getTime().toUnix()).writeFile execProcess benchCmd - serverProcess.close() - else: - echo "Invalid option: ", paramStr(1) \ No newline at end of file diff --git a/example.nim b/example.nim index 6ea8d8c..fe86f5a 100644 --- a/example.nim +++ b/example.nim @@ -7,36 +7,35 @@ import strformat # "/" -> get: - ctx.send "Mike is running!" + ctx.send "Mike is running!" "/hello" -> post: - let names = ctx.json(seq[string]) - for name in names: - echo name - ctx.send "OK" + let names = ctx.json(seq[string]) + for name in names: + echo name + ctx.send "OK" "/shutdown" -> get: - quit 0 + quit 0 # # Custom data # type - Person = ref object of RootObj - name: string - age: int + Person = ref object of RootObj + name: string + age: int "/person/:name/:age" -> beforeGet: - echo ctx.pathParams - ctx &= Person( - name: ctx.pathParams["name"], - age: ctx.pathParams["age"].parseInt() - ) + ctx &= Person( + name: ctx.pathParams["name"], + age: ctx.pathParams["age"].parseInt() + ) "/person/:name/:age" -> get: - let person = ctx[Person] - ctx.send fmt"Hello {person.name} aged {person.age}" + let person = ctx[Person] + ctx.send fmt"Hello {person.name} aged {person.age}" run() diff --git a/mike.nimble b/mike.nimble index 3006f2d..ffdf031 100644 --- a/mike.nimble +++ b/mike.nimble @@ -17,7 +17,3 @@ requires "httpx >= 0.3.7" task ex, "Runs the example": selfExec "c -f -d:debug -r example" - -task bench, "Runs a benchmark and saves it to a file with the current time": - for cmd in ["compile", "bench"]: - selfExec "r -d:release --gc:arc --opt:size benchmark " & cmd