From 2ea4a0e898414577846e4216bec69cf78fa3654e Mon Sep 17 00:00:00 2001 From: Benjamin Cane Date: Sun, 7 Jul 2024 07:57:36 -0700 Subject: [PATCH 1/3] Restructuring Benchmarks --- .github/workflows/benchmarks.yml | 29 +++++- benchmarks/benchmark_test.go | 156 ---------------------------- benchmarks/go.mod | 33 ------ benchmarks/go.sum | 69 ------------ drivers/bbolt/benchmark_test.go | 95 +++++++++++++++++ drivers/cassandra/benchmark_test.go | 111 ++++++++++++++++++++ drivers/hashmap/benchmark_test.go | 91 ++++++++++++++++ drivers/nats/benchmark_test.go | 90 ++++++++++++++++ drivers/redis/benchmark_test.go | 120 +++++++++++++++++++++ 9 files changed, 533 insertions(+), 261 deletions(-) delete mode 100644 benchmarks/benchmark_test.go delete mode 100644 benchmarks/go.mod delete mode 100644 benchmarks/go.sum create mode 100644 drivers/bbolt/benchmark_test.go create mode 100644 drivers/cassandra/benchmark_test.go create mode 100644 drivers/hashmap/benchmark_test.go create mode 100644 drivers/nats/benchmark_test.go create mode 100644 drivers/redis/benchmark_test.go diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 9f9b530..c0db206 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -45,7 +45,8 @@ jobs: - name: Execute Benchmarks run: | sleep 120 - /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_Cassandra" ./... + cd ./drivers/cassandra + /usr/local/go/bin/go test -run=Benchmark -bench ./... | tee output.txt redis: runs-on: ubuntu-latest @@ -69,6 +70,7 @@ jobs: - name: Execute Benchmarks run: | sleep 60 + cd ./drivers/redis /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_Redis" ./... keydb: @@ -86,6 +88,7 @@ jobs: - name: Execute Benchmarks run: | sleep 60 + cd ./drivers/redis /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_KeyDB" ./... nats: @@ -108,6 +111,7 @@ jobs: - name: Execute Benchmarks run: | sleep 60 + cd ./drivers/nats /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_NATS" ./... hashmap: @@ -120,7 +124,24 @@ jobs: run: | curl -L https://go.dev/dl/go1.22.0.linux-amd64.tar.gz | tar -C /usr/local -xzf - - name: Execute Benchmarks - run: /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_Hashmap" ./... + run: | + cd ./drivers/hashmap + /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_Hashmap" ./... | tee output.txt + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: hashmap-benchmark + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'go' + output-file-path: output.txt + external-data-json-path: ./cache/hashmap-benchmark-data.json + fail-on-alert: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-on-alert: true + summary-always: true boltdb: runs-on: ubuntu-latest @@ -132,4 +153,6 @@ jobs: run: | curl -L https://go.dev/dl/go1.22.0.linux-amd64.tar.gz | tar -C /usr/local -xzf - - name: Execute Benchmarks - run: /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_BoltDB" ./... + run: | + cd ./drivers/bbolt + /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_BoltDB" ./... diff --git a/benchmarks/benchmark_test.go b/benchmarks/benchmark_test.go deleted file mode 100644 index f72e835..0000000 --- a/benchmarks/benchmark_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package benchmarks - -import ( - "fmt" - "os" - "testing" - "time" - - "github.com/tarmac-project/hord" - "github.com/tarmac-project/hord/drivers/bbolt" - "github.com/tarmac-project/hord/drivers/cassandra" - "github.com/tarmac-project/hord/drivers/hashmap" - "github.com/tarmac-project/hord/drivers/nats" - "github.com/tarmac-project/hord/drivers/redis" -) - -func BenchmarkDrivers(b *testing.B) { - // Create some test data for Benchmarks - data := []byte(` - { - "userId": 1, - "id": 1, - "title": "sunt aut facere repellat provident occaecati excepturi optio reprehenderit", - "body": "quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto" - } - `) - - // Create a Set of drivers to benchmark - drivers := []string{"Redis", "Cassandra", "Hashmap", "BoltDB", "NATS", "Dragonfly", "KeyDB"} - - // Loop through the various DBs and TestData - for _, driver := range drivers { - b.Run("Bench_"+driver, func(b *testing.B) { - var db hord.Database - var err error - switch driver { - case "Redis", "Dragonfly", "KeyDB": - server := "redis:6379" - // Connect to Dragonfly - if driver == "Dragonfly" { - server = "dragonfly:6379" - } - // Connect to KeyDB - if driver == "KeyDB" { - server = "keydb:6379" - } - - // Connect to Redis - db, err = redis.Dial(redis.Config{ - ConnectTimeout: time.Duration(5) * time.Second, - MaxActive: 500, - MaxIdle: 100, - IdleTimeout: time.Duration(5) * time.Second, - Server: server, - }) - if err != nil { - b.Fatalf("Got unexpected error when connecting to Redis - %s", err) - } - - case "Cassandra": - // Connect to Cassandra - hosts := []string{"cassandra-primary", "cassandra"} - db, err = cassandra.Dial(cassandra.Config{Hosts: hosts, Keyspace: "hord"}) - if err != nil { - b.Fatalf("Got unexpected error when connecting to a cassandra cluster - %s", err) - } - - case "NATS": - db, err = nats.Dial(nats.Config{URL: "nats", Bucket: "hord"}) - if err != nil { - b.Fatalf("Got unexpected error when connecting to nats - %s", err) - } - - case "Hashmap": - db, err = hashmap.Dial(hashmap.Config{}) - if err != nil { - b.Fatalf("Got unexpected error when initializing hashmap - %s", err) - } - - case "BoltDB": - db, err = bbolt.Dial(bbolt.Config{ - Bucketname: "test", - Filename: "/tmp/bbolt-benchmark.db", - }) - if err != nil { - b.Fatalf("Got unexpected error when initializing hashmap - %s", err) - } - defer os.Remove("/tmp/bbolt-benchmark.db") - - default: - b.Fatalf("Unknown DB Driver Specified") - } - defer db.Close() - - // Setup DB - err = db.Setup() - if err != nil { - b.Fatalf("Unknown error setting up DB - %s", err) - } - - // Execute HealthCheck - err = db.HealthCheck() - if err != nil { - b.Fatalf("Error while checking health of DB - %s", err) - } - - b.Run("SET", func(b *testing.B) { - // Clean up Keys Created for Test - b.Cleanup(func() { - keys, _ := db.Keys() - for _, d := range keys { - _ = db.Delete(d) - } - }) - - // Exec Benchmark - for i := 0; i < b.N; i++ { - err := db.Set("Test_Keys_"+fmt.Sprintf("%d", i), data) - if err != nil { - b.Fatalf("Error when executing Benchmark test - %s", err) - } - } - }) - - b.Run("GET", func(b *testing.B) { - // Clean up Keys Created for Test - b.Cleanup(func() { - keys, _ := db.Keys() - for _, d := range keys { - _ = db.Delete(d) - } - }) - - // Setup A Bunch of Keys - b.StopTimer() - for i := 0; i < 5000; i++ { - _ = db.Set("Test_Keys_"+fmt.Sprintf("%d", i), data) - } - - // Exec Benchmark - count := 0 - b.StartTimer() - for i := 0; i < b.N; i++ { - if count > 4999 { - count = 0 - } - _, err := db.Get("Test_Keys_" + fmt.Sprintf("%d", count)) - if err != nil { - b.Fatalf("Error when executing Benchmark test - %s", err) - } - } - }) - - }) - } -} diff --git a/benchmarks/go.mod b/benchmarks/go.mod deleted file mode 100644 index 230a758..0000000 --- a/benchmarks/go.mod +++ /dev/null @@ -1,33 +0,0 @@ -module github.com/tarmac-project/hord/benchmarks - -go 1.21 - -toolchain go1.22.4 - -require ( - github.com/tarmac-project/hord v0.4.0 - github.com/tarmac-project/hord/drivers/bbolt v0.4.0 - github.com/tarmac-project/hord/drivers/cassandra v0.4.0 - github.com/tarmac-project/hord/drivers/hashmap v0.4.0 - github.com/tarmac-project/hord/drivers/nats v0.4.0 - github.com/tarmac-project/hord/drivers/redis v0.4.0 -) - -require ( - github.com/FZambia/sentinel v1.1.1 // indirect - github.com/gocql/gocql v1.6.0 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/gomodule/redigo v1.9.2 // indirect - github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/klauspost/compress v1.17.7 // indirect - github.com/madflojo/hord v0.3.0 // indirect - github.com/nats-io/nats.go v1.36.0 // indirect - github.com/nats-io/nkeys v0.4.7 // indirect - github.com/nats-io/nuid v1.0.1 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect - go.etcd.io/bbolt v1.3.10 // indirect - golang.org/x/crypto v0.20.0 // indirect - golang.org/x/sys v0.17.0 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) diff --git a/benchmarks/go.sum b/benchmarks/go.sum deleted file mode 100644 index a9e554e..0000000 --- a/benchmarks/go.sum +++ /dev/null @@ -1,69 +0,0 @@ -github.com/FZambia/sentinel v1.1.1 h1:0ovTimlR7Ldm+wR15GgO+8C2dt7kkn+tm3PQS+Qk3Ek= -github.com/FZambia/sentinel v1.1.1/go.mod h1:ytL1Am/RLlAoAXG6Kj5LNuw/TRRQrv2rt2FT26vP5gI= -github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= -github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= -github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s= -github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw= -github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= -github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/madflojo/hord v0.3.0 h1:uq1u2EiOhKhiA7vSLm22I54MxUOTYaD0A8Btf62+iwM= -github.com/madflojo/hord v0.3.0/go.mod h1:wXy4XFyefuQIRSuVyNQJkfJVDRpXGv23yElKuBQUdoY= -github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU= -github.com/nats-io/nats.go v1.36.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= -github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= -github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= -github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tarmac-project/hord v0.4.0 h1:8ezVc4PULcr9lDWX8zaZaCIunWR1ef50Lz88o7CzkaM= -github.com/tarmac-project/hord v0.4.0/go.mod h1:b46vLVFfL9G/WG5BYNTNoordvQ9wQ+ibs1/Fn4v0vkE= -github.com/tarmac-project/hord/drivers/bbolt v0.4.0 h1:Od20mgOrgS/DzoWUJsk9m011RXJ7tB95S1s2PfyY+ek= -github.com/tarmac-project/hord/drivers/bbolt v0.4.0/go.mod h1:EwF2eCiILhwKaRfvqMJlm0Ffafh7ngQBsqPag4qvACo= -github.com/tarmac-project/hord/drivers/cassandra v0.4.0 h1:E9OF86LGAMhV4yRCevlIrAcPMl4Kx17iTngoUP9cdZI= -github.com/tarmac-project/hord/drivers/cassandra v0.4.0/go.mod h1:BmELPMm+KgOXiMj8iKfa6U5CIpEDlCWe4uVpXqLIf1o= -github.com/tarmac-project/hord/drivers/hashmap v0.4.0 h1:n0Cb9V/KfSIyjX7KuXbgypT7O53flLwwt4jPIJ5UJAA= -github.com/tarmac-project/hord/drivers/hashmap v0.4.0/go.mod h1:67seMbQzJ4fyhR93BoM9TYkfaememCDpWY6HZCNJCzk= -github.com/tarmac-project/hord/drivers/nats v0.4.0 h1:vvshMDxXUjyDwJhWkhwupgIXQa8B4pMcgguQsI6Om2g= -github.com/tarmac-project/hord/drivers/nats v0.4.0/go.mod h1:L+xFlqeVa1bsabSySz0TuCz0RiHgpA14WIBFi0yWYgc= -github.com/tarmac-project/hord/drivers/redis v0.4.0 h1:Pj3oWzS0oOD6EKdqFdADzBmOFaxCJuV18X/jngyo1qE= -github.com/tarmac-project/hord/drivers/redis v0.4.0/go.mod h1:Wak3REtuIOcjaCHnrMKBPEhkndtFsRWzZKqrjwPFIsk= -go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= -go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= -golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg= -golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/drivers/bbolt/benchmark_test.go b/drivers/bbolt/benchmark_test.go new file mode 100644 index 0000000..948cf02 --- /dev/null +++ b/drivers/bbolt/benchmark_test.go @@ -0,0 +1,95 @@ +package bbolt + +import ( + "fmt" + "os" + "testing" + + "github.com/tarmac-project/hord" +) + +func BenchmarkDrivers(b *testing.B) { + // Create some test data for Benchmarks + data := []byte(` + { + "userId": 1, + "id": 1, + "title": "sunt aut facere repellat provident occaecati excepturi optio reprehenderit", + "body": "quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto" + } + `) + + b.Run("Bench_BBolt", func(b *testing.B) { + var db hord.Database + var err error + db, err = Dial(Config{ + Bucketname: "test", + Filename: "/tmp/bbolt-benchmark.db", + }) + if err != nil { + b.Fatalf("Got unexpected error when initializing hashmap - %s", err) + } + defer os.Remove("/tmp/bbolt-benchmark.db") + defer db.Close() + + // Setup DB + err = db.Setup() + if err != nil { + b.Fatalf("Unknown error setting up DB - %s", err) + } + + // Execute HealthCheck + err = db.HealthCheck() + if err != nil { + b.Fatalf("Error while checking health of DB - %s", err) + } + + b.Run("SET", func(b *testing.B) { + // Clean up Keys Created for Test + b.Cleanup(func() { + keys, _ := db.Keys() + for _, d := range keys { + _ = db.Delete(d) + } + }) + + // Exec Benchmark + for i := 0; i < b.N; i++ { + err := db.Set("Test_Keys_"+fmt.Sprintf("%d", i), data) + if err != nil { + b.Fatalf("Error when executing Benchmark test - %s", err) + } + } + }) + + b.Run("GET", func(b *testing.B) { + // Clean up Keys Created for Test + b.Cleanup(func() { + keys, _ := db.Keys() + for _, d := range keys { + _ = db.Delete(d) + } + }) + + // Setup A Bunch of Keys + b.StopTimer() + for i := 0; i < 5000; i++ { + _ = db.Set("Test_Keys_"+fmt.Sprintf("%d", i), data) + } + + // Exec Benchmark + count := 0 + b.StartTimer() + for i := 0; i < b.N; i++ { + if count > 4999 { + count = 0 + } + _, err := db.Get("Test_Keys_" + fmt.Sprintf("%d", count)) + if err != nil { + b.Fatalf("Error when executing Benchmark test - %s", err) + } + } + }) + + }) +} diff --git a/drivers/cassandra/benchmark_test.go b/drivers/cassandra/benchmark_test.go new file mode 100644 index 0000000..10074fb --- /dev/null +++ b/drivers/cassandra/benchmark_test.go @@ -0,0 +1,111 @@ +package cassandra + +import ( + "fmt" + "testing" + + "github.com/tarmac-project/hord" +) + +func BenchmarkDrivers(b *testing.B) { + // Create some test data for Benchmarks + data := []byte(` + { + "userId": 1, + "id": 1, + "title": "sunt aut facere repellat provident occaecati excepturi optio reprehenderit", + "body": "quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto" + } + `) + + // Create a Set of datastores to benchmark (TODO: Add more Cassandra Compliant DBs) + datastores := []string{"Cassandra"} + + // Loop through the various DBs and TestData + for _, datastore := range datastores { + b.Run("Bench_"+datastore, func(b *testing.B) { + var db hord.Database + var err error + + // Base Config + cfg := Config{ + Keyspace: "hord", + } + + // Set Hosts based on DB + switch datastore { + case "Cassandra": + cfg.Hosts = []string{"cassandra-primary", "cassandra"} + default: + b.Fatalf("Unknown DB Driver Specified") + } + + // Connect to DB + db, err = Dial(cfg) + if err != nil { + b.Fatalf("Got unexpected error when connecting to a cassandra cluster - %s", err) + } + defer db.Close() + + // Setup DB + err = db.Setup() + if err != nil { + b.Fatalf("Unknown error setting up DB - %s", err) + } + + // Execute HealthCheck + err = db.HealthCheck() + if err != nil { + b.Fatalf("Error while checking health of DB - %s", err) + } + + b.Run("SET", func(b *testing.B) { + // Clean up Keys Created for Test + b.Cleanup(func() { + keys, _ := db.Keys() + for _, d := range keys { + _ = db.Delete(d) + } + }) + + // Exec Benchmark + for i := 0; i < b.N; i++ { + err := db.Set("Test_Keys_"+fmt.Sprintf("%d", i), data) + if err != nil { + b.Fatalf("Error when executing Benchmark test - %s", err) + } + } + }) + + b.Run("GET", func(b *testing.B) { + // Clean up Keys Created for Test + b.Cleanup(func() { + keys, _ := db.Keys() + for _, d := range keys { + _ = db.Delete(d) + } + }) + + // Setup A Bunch of Keys + b.StopTimer() + for i := 0; i < 5000; i++ { + _ = db.Set("Test_Keys_"+fmt.Sprintf("%d", i), data) + } + + // Exec Benchmark + count := 0 + b.StartTimer() + for i := 0; i < b.N; i++ { + if count > 4999 { + count = 0 + } + _, err := db.Get("Test_Keys_" + fmt.Sprintf("%d", count)) + if err != nil { + b.Fatalf("Error when executing Benchmark test - %s", err) + } + } + }) + + }) + } +} diff --git a/drivers/hashmap/benchmark_test.go b/drivers/hashmap/benchmark_test.go new file mode 100644 index 0000000..d0fab33 --- /dev/null +++ b/drivers/hashmap/benchmark_test.go @@ -0,0 +1,91 @@ +package hashmap + +import ( + "fmt" + "testing" + + "github.com/tarmac-project/hord" +) + +func BenchmarkDrivers(b *testing.B) { + // Create some test data for Benchmarks + data := []byte(` + { + "userId": 1, + "id": 1, + "title": "sunt aut facere repellat provident occaecati excepturi optio reprehenderit", + "body": "quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto" + } + `) + + // Loop through the various DBs and TestData + b.Run("Bench_Hashmap", func(b *testing.B) { + var db hord.Database + var err error + db, err = Dial(Config{}) + if err != nil { + b.Fatalf("Got unexpected error when initializing hashmap - %s", err) + } + defer db.Close() + + // Setup DB + err = db.Setup() + if err != nil { + b.Fatalf("Unknown error setting up DB - %s", err) + } + + // Execute HealthCheck + err = db.HealthCheck() + if err != nil { + b.Fatalf("Error while checking health of DB - %s", err) + } + + b.Run("SET", func(b *testing.B) { + // Clean up Keys Created for Test + b.Cleanup(func() { + keys, _ := db.Keys() + for _, d := range keys { + _ = db.Delete(d) + } + }) + + // Exec Benchmark + for i := 0; i < b.N; i++ { + err := db.Set("Test_Keys_"+fmt.Sprintf("%d", i), data) + if err != nil { + b.Fatalf("Error when executing Benchmark test - %s", err) + } + } + }) + + b.Run("GET", func(b *testing.B) { + // Clean up Keys Created for Test + b.Cleanup(func() { + keys, _ := db.Keys() + for _, d := range keys { + _ = db.Delete(d) + } + }) + + // Setup A Bunch of Keys + b.StopTimer() + for i := 0; i < 5000; i++ { + _ = db.Set("Test_Keys_"+fmt.Sprintf("%d", i), data) + } + + // Exec Benchmark + count := 0 + b.StartTimer() + for i := 0; i < b.N; i++ { + if count > 4999 { + count = 0 + } + _, err := db.Get("Test_Keys_" + fmt.Sprintf("%d", count)) + if err != nil { + b.Fatalf("Error when executing Benchmark test - %s", err) + } + } + }) + + }) +} diff --git a/drivers/nats/benchmark_test.go b/drivers/nats/benchmark_test.go new file mode 100644 index 0000000..afdcf73 --- /dev/null +++ b/drivers/nats/benchmark_test.go @@ -0,0 +1,90 @@ +package nats + +import ( + "fmt" + "testing" + + "github.com/tarmac-project/hord" +) + +func BenchmarkDrivers(b *testing.B) { + // Create some test data for Benchmarks + data := []byte(` + { + "userId": 1, + "id": 1, + "title": "sunt aut facere repellat provident occaecati excepturi optio reprehenderit", + "body": "quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto" + } + `) + + b.Run("Bench_NATS", func(b *testing.B) { + var db hord.Database + var err error + db, err = Dial(Config{URL: "nats", Bucket: "hord"}) + if err != nil { + b.Fatalf("Got unexpected error when connecting to nats - %s", err) + } + defer db.Close() + + // Setup DB + err = db.Setup() + if err != nil { + b.Fatalf("Unknown error setting up DB - %s", err) + } + + // Execute HealthCheck + err = db.HealthCheck() + if err != nil { + b.Fatalf("Error while checking health of DB - %s", err) + } + + b.Run("SET", func(b *testing.B) { + // Clean up Keys Created for Test + b.Cleanup(func() { + keys, _ := db.Keys() + for _, d := range keys { + _ = db.Delete(d) + } + }) + + // Exec Benchmark + for i := 0; i < b.N; i++ { + err := db.Set("Test_Keys_"+fmt.Sprintf("%d", i), data) + if err != nil { + b.Fatalf("Error when executing Benchmark test - %s", err) + } + } + }) + + b.Run("GET", func(b *testing.B) { + // Clean up Keys Created for Test + b.Cleanup(func() { + keys, _ := db.Keys() + for _, d := range keys { + _ = db.Delete(d) + } + }) + + // Setup A Bunch of Keys + b.StopTimer() + for i := 0; i < 5000; i++ { + _ = db.Set("Test_Keys_"+fmt.Sprintf("%d", i), data) + } + + // Exec Benchmark + count := 0 + b.StartTimer() + for i := 0; i < b.N; i++ { + if count > 4999 { + count = 0 + } + _, err := db.Get("Test_Keys_" + fmt.Sprintf("%d", count)) + if err != nil { + b.Fatalf("Error when executing Benchmark test - %s", err) + } + } + }) + + }) +} diff --git a/drivers/redis/benchmark_test.go b/drivers/redis/benchmark_test.go new file mode 100644 index 0000000..54319a0 --- /dev/null +++ b/drivers/redis/benchmark_test.go @@ -0,0 +1,120 @@ +package redis + +import ( + "fmt" + "testing" + "time" + + "github.com/tarmac-project/hord" +) + +func BenchmarkDrivers(b *testing.B) { + // Create some test data for Benchmarks + data := []byte(` + { + "userId": 1, + "id": 1, + "title": "sunt aut facere repellat provident occaecati excepturi optio reprehenderit", + "body": "quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto" + } + `) + + // Create a Set of datastores to benchmark + datastores := []string{"Redis", "Dragonfly", "KeyDB"} + + // Loop through the various DBs and TestData + for _, datastore := range datastores { + b.Run("Bench_"+datastore, func(b *testing.B) { + var db hord.Database + var err error + + // Base Config + cfg := Config{ + ConnectTimeout: time.Duration(5) * time.Second, + MaxActive: 500, + MaxIdle: 100, + IdleTimeout: time.Duration(5) * time.Second, + } + + // Set Server Based on Datastore + switch datastore { + case "Redis": + cfg.Server = "redis:6379" + case "Dragonfly": + cfg.Server = "dragonfly:6379" + case "KeyDB": + cfg.Server = "keydb:6379" + default: + b.Fatalf("Unknown DB Specified") + } + + // Connect to Redis + db, err = Dial(cfg) + if err != nil { + b.Fatalf("Got unexpected error when connecting to Redis - %s", err) + } + + defer db.Close() + + // Setup DB + err = db.Setup() + if err != nil { + b.Fatalf("Unknown error setting up DB - %s", err) + } + + // Execute HealthCheck + err = db.HealthCheck() + if err != nil { + b.Fatalf("Error while checking health of DB - %s", err) + } + + b.Run("SET", func(b *testing.B) { + // Clean up Keys Created for Test + b.Cleanup(func() { + keys, _ := db.Keys() + for _, d := range keys { + _ = db.Delete(d) + } + }) + + // Exec Benchmark + for i := 0; i < b.N; i++ { + err := db.Set("Test_Keys_"+fmt.Sprintf("%d", i), data) + if err != nil { + b.Fatalf("Error when executing Benchmark test - %s", err) + } + } + }) + + b.Run("GET", func(b *testing.B) { + // Clean up Keys Created for Test + b.Cleanup(func() { + keys, _ := db.Keys() + for _, d := range keys { + _ = db.Delete(d) + } + }) + + // Setup A Bunch of Keys + b.StopTimer() + for i := 0; i < 5000; i++ { + _ = db.Set("Test_Keys_"+fmt.Sprintf("%d", i), data) + } + + // Exec Benchmark + count := 0 + b.StartTimer() + for i := 0; i < b.N; i++ { + if count > 4999 { + count = 0 + } + _, err := db.Get("Test_Keys_" + fmt.Sprintf("%d", count)) + if err != nil { + b.Fatalf("Error when executing Benchmark test - %s", err) + } + } + }) + + }) + } +} From d740cdaefe75bec07dfa8f9bf497e8d1ae0099ee Mon Sep 17 00:00:00 2001 From: Benjamin Cane Date: Sun, 7 Jul 2024 08:04:55 -0700 Subject: [PATCH 2/3] Fixing benchmark measurements --- .github/workflows/benchmarks.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index c0db206..c9708bb 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -8,6 +8,11 @@ on: - actions - main pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: cassandra: runs-on: ubuntu-latest @@ -136,7 +141,7 @@ jobs: uses: benchmark-action/github-action-benchmark@v1 with: tool: 'go' - output-file-path: output.txt + output-file-path: ./drivers/hashmap/output.txt external-data-json-path: ./cache/hashmap-benchmark-data.json fail-on-alert: true github-token: ${{ secrets.GITHUB_TOKEN }} From 214772340a70c230381f05a0838a99ba7ed015ea Mon Sep 17 00:00:00 2001 From: Benjamin Cane Date: Sun, 7 Jul 2024 08:22:21 -0700 Subject: [PATCH 3/3] Adding Benchmark Summaries for all Jobs --- .github/workflows/benchmarks.yml | 83 ++++++++++++++++++++++++++++++-- drivers/bbolt/benchmark_test.go | 2 +- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index c9708bb..ef5a5c6 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -52,6 +52,21 @@ jobs: sleep 120 cd ./drivers/cassandra /usr/local/go/bin/go test -run=Benchmark -bench ./... | tee output.txt + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: cassandra-benchmark + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'go' + output-file-path: ./drivers/cassandra/output.txt + external-data-json-path: ./cache/cassandra-benchmark-data.json + fail-on-alert: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-on-alert: true + summary-always: true redis: runs-on: ubuntu-latest @@ -76,7 +91,22 @@ jobs: run: | sleep 60 cd ./drivers/redis - /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_Redis" ./... + /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_Redis" ./... | tee output.txt + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: redis-benchmark + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'go' + output-file-path: ./drivers/redis/output.txt + external-data-json-path: ./cache/redis-benchmark-data.json + fail-on-alert: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-on-alert: true + summary-always: true keydb: runs-on: ubuntu-latest @@ -94,7 +124,22 @@ jobs: run: | sleep 60 cd ./drivers/redis - /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_KeyDB" ./... + /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_KeyDB" ./... | tee output.txt + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: keydb-benchmark + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'go' + output-file-path: ./drivers/redis/output.txt + external-data-json-path: ./cache/keydb-benchmark-data.json + fail-on-alert: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-on-alert: true + summary-always: true nats: runs-on: ubuntu-latest @@ -117,7 +162,22 @@ jobs: run: | sleep 60 cd ./drivers/nats - /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_NATS" ./... + /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_NATS" ./... | tee output.txt + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: nats-benchmark + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'go' + output-file-path: ./drivers/nats/output.txt + external-data-json-path: ./cache/nats-benchmark-data.json + fail-on-alert: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-on-alert: true + summary-always: true hashmap: runs-on: ubuntu-latest @@ -160,4 +220,19 @@ jobs: - name: Execute Benchmarks run: | cd ./drivers/bbolt - /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_BoltDB" ./... + /usr/local/go/bin/go test -run=Benchmark -bench "BenchmarkDrivers/Bench_BoltDB" ./... | tee output.txt + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: boltdb-benchmark + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'go' + output-file-path: ./drivers/bbolt/output.txt + external-data-json-path: ./cache/boltdb-benchmark-data.json + fail-on-alert: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-on-alert: true + summary-always: true diff --git a/drivers/bbolt/benchmark_test.go b/drivers/bbolt/benchmark_test.go index 948cf02..731e50f 100644 --- a/drivers/bbolt/benchmark_test.go +++ b/drivers/bbolt/benchmark_test.go @@ -19,7 +19,7 @@ func BenchmarkDrivers(b *testing.B) { } `) - b.Run("Bench_BBolt", func(b *testing.B) { + b.Run("Bench_BoltDB", func(b *testing.B) { var db hord.Database var err error db, err = Dial(Config{