Merge pull request #802 from threefoldtech/development_add_cache_warmer

Add relay cache warmer

.goreleaser.yaml

@@ -112,6 +112,20 @@ builds:
       - -X github.com/threefoldtech/tfgrid-sdk-go/monitoring-bot/cmd.version={{.Tag}}
       - -X github.com/threefoldtech/tfgrid-sdk-go/monitoring-bot/cmd.commit={{.Commit}}
 
+  - dir: ./tools/relay-cache-warmer
+    goos:
+      - linux
+      - windows
+      - darwin
+    binary: cache-warmer
+    id: cache-warmer
+
+    ignore:
+      - goos: windows
+    ldflags:
+      - -X main.version={{.Tag}}
+      - -X main.commit={{.Commit}}
+
 archives:
   - format: tar.gz
     # this name template makes the OS and Arch compatible with the results of uname.

go.work

@@ -11,4 +11,5 @@ use (
 	./monitoring-bot
 	./rmb-sdk-go
 	./user-contracts-mon
+	./tools/relay-cache-warmer
 )

tools/relay-cache-warmer/Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.21-alpine as builder
+
+WORKDIR /app
+
+COPY . .
+
+RUN go build -o bin/cache-warmer .
+
+FROM alpine:3.19
+
+COPY --from=builder /app/bin/cache-warmer .
+
+ENTRYPOINT [ "/cache-warmer" ]

tools/relay-cache-warmer/Makefile

@@ -0,0 +1,22 @@
+GOPATH := $(shell go env GOPATH)
+
+all: verifiers
+
+getverifiers:
+	@echo "Installing golangci-lint" && go install github.com/golangci/golangci-lint/cmd/golangci-lint
+	go mod tidy
+
+lint:
+	@echo "Running $@"
+	golangci-lint run -c ../.golangci.yml
+
+build:
+	@echo "Running $@"
+	@go build -ldflags=\
+	"-X 'main.commit=$(shell git rev-parse HEAD)'\
+	 -X 'main.version=$(shell git tag --sort=-version:refname | head -n 1)'"\
+	 -o bin/cache-warmer .
+
+clean:
+	rm ./bin -rf
+

tools/relay-cache-warmer/README.md

@@ -0,0 +1,35 @@
+# Relay Cache Warmer
+
+Relay Cache Warmer is a software used to warm Relay's Redis cache with twins fetched from GraphQl periodically to avoid the Relay slowdown due to fetching twins from TFChain on RMB calls.
+
+## Usage
+
+Run:
+
+```bash
+cache-warmer --interval 10 --graphql https://graphql.grid.tf/graphql --redis-url redis://localhost:6379
+```
+
+## Build
+
+You need Go(1.21) and make.
+Run:
+
+```bash
+make build
+```
+
+## Flags
+
+```text
+  --graphql string
+        graphql url (default "https://graphql.grid.tf/graphql")
+  --interval uint
+        cache warming interval (default 10)
+  --redis-password string
+        redis password
+  --redis-url string
+        redis url (default "redis://localhost:6379")
+  --version
+        print version and exit
+```

tools/relay-cache-warmer/go.mod

@@ -0,0 +1,5 @@
+module github.com/threefoldtech/tfgrid-sdk-go/relay-cache-warmer
+
+go 1.21
+
+require github.com/gomodule/redigo v1.8.9

tools/relay-cache-warmer/go.sum

@@ -0,0 +1,12 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
+github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

tools/relay-cache-warmer/graphql.go

@@ -0,0 +1,187 @@
+package main
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gomodule/redigo/redis"
+)
+
+const query = `query TwinsQuery {
+  items: twinsConnection(first: %d, orderBy: id_ASC%s) {
+    pageInfo {
+  	  endCursor
+  	  hasNextPage
+    }
+  }
+  twins(limit: %d, offset: %s, orderBy: id_ASC) {
+    accountID
+    publicKey
+    twinID
+    relay
+  }
+}`
+
+const (
+	twinsPerPage     = 500
+	requestsInterval = 3 * time.Second
+)
+
+type Twin struct {
+	ID        uint      `json:"id"`
+	AccountID string    `json:"account"`
+	Relay     []string  `json:"relay"`
+	PK        JSONArray `json:"pk"`
+}
+
+// only exists because json marshals []byte to a string instead of json array
+type JSONArray []byte
+
+func (u JSONArray) MarshalJSON() ([]byte, error) {
+	if u == nil || len(u) == 0 {
+		return []byte("null"), nil
+	}
+
+	var buf bytes.Buffer
+	_, _ = buf.WriteRune('[')
+	for i, b := range u {
+		if i != 0 {
+			_, _ = buf.WriteRune(',')
+		}
+		_, _ = buf.WriteString(strconv.FormatUint(uint64(b), 10))
+	}
+	_, _ = buf.WriteRune(']')
+
+	return buf.Bytes(), nil
+
+}
+
+func warmTwins(pool *redis.Pool, graphql string) error {
+	offset := "0"
+
+	for {
+		afterCondition := ""
+		if offset != "0" && offset != "" {
+			afterCondition = fmt.Sprintf(`, after:"%s"`, offset)
+		}
+		body := fmt.Sprintf(
+			query,
+			twinsPerPage,
+			afterCondition,
+			twinsPerPage,
+			offset,
+		)
+		pagination, gtwins, err := queryGraphql(graphql, body)
+		if err != nil {
+			return err
+		}
+		twins, err := graphqlTwinsToRelayTwins(gtwins)
+		if err != nil {
+			return err
+		}
+		if !pagination.PageInfo.HasNextPage {
+			break
+		}
+		offset = pagination.PageInfo.EndCursor
+
+		err = writeTwins(pool, twins)
+		if err != nil {
+			return err
+		}
+		time.Sleep(requestsInterval)
+	}
+
+	return nil
+}
+
+func queryGraphql(graphql, body string) (paginationData, []graphqlTwin, error) {
+	bodyBytes, err := json.Marshal(map[string]interface{}{"query": body})
+	if err != nil {
+		return paginationData{}, nil, err
+	}
+	reader := bytes.NewReader(bodyBytes)
+	resp, err := http.Post(graphql, "application/json", reader)
+	if err != nil {
+		return paginationData{}, nil, err
+	}
+	defer resp.Body.Close()
+	r, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return paginationData{}, nil, err
+	}
+	respBody := map[string]interface{}{}
+	err = json.Unmarshal(r, &respBody)
+	if err != nil {
+		return paginationData{}, nil, err
+	}
+	var gTwins []graphqlTwin
+	data, ok := respBody["data"].(map[string]interface{})
+	if !ok {
+		return paginationData{}, nil, fmt.Errorf("got unexpected format %s", string(r))
+	}
+
+	twins := data["twins"]
+	dataBytes, err := json.Marshal(twins)
+	if err != nil {
+		return paginationData{}, nil, err
+	}
+	if err := json.Unmarshal(dataBytes, &gTwins); err != nil {
+		return paginationData{}, nil, err
+	}
+	items := data["items"]
+	itemsBytes, err := json.Marshal(items)
+	if err != nil {
+		return paginationData{}, nil, err
+	}
+	var pdata paginationData
+
+	if err := json.Unmarshal(itemsBytes, &pdata); err != nil {
+		return paginationData{}, nil, err
+	}
+	return pdata, gTwins, nil
+}
+
+func graphqlTwinsToRelayTwins(gTwins []graphqlTwin) ([]Twin, error) {
+	twins := make([]Twin, 0, len(gTwins))
+	for _, gTwin := range gTwins {
+		twin := Twin{
+			ID:        gTwin.ID,
+			AccountID: gTwin.AccountID,
+		}
+		if gTwin.Relay != nil && len(*gTwin.Relay) != 0 {
+			twin.Relay = strings.Split(*gTwin.Relay, "_")
+		}
+		if gTwin.PK != nil && len(*gTwin.PK) != 0 {
+			pk, err := hex.DecodeString(strings.TrimPrefix(*gTwin.PK, "0x"))
+			if err != nil {
+				return nil, err
+			}
+			twin.PK = pk
+		}
+		twins = append(twins, twin)
+	}
+	return twins, nil
+}
+
+type paginationData struct {
+	Count    int      `json:"count"`
+	PageInfo PageInfo `json:"pageInfo"`
+}
+type PageInfo struct {
+	EndCursor   string `json:"endCursor"`
+	HasNextPage bool   `json:"hasNextPage"`
+}
+
+type graphqlTwin struct {
+	ID        uint    `json:"twinID"`
+	AccountID string  `json:"accountID"`
+	Relay     *string `json:"relay"`
+	PK        *string `json:"publicKey"`
+}

tools/relay-cache-warmer/main.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/gomodule/redigo/redis"
+)
+
+// set at build time
+var commit string
+var version string
+
+func main() {
+	var graphql string
+	var interval uint64
+	var redisURL string
+	var redisPassword string
+	var versionFlag bool
+	flag.StringVar(&graphql, "graphql", "https://graphql.grid.tf/graphql", "graphql url")
+	flag.Uint64Var(&interval, "interval", 10, "cache warming interval")
+	flag.StringVar(&redisURL, "redis-url", "redis://localhost:6379", "redis url")
+	flag.StringVar(&redisPassword, "redis-password", "", "redis password")
+	flag.BoolVar(&versionFlag, "version", false, "print version and exit")
+
+	flag.Parse()
+	if versionFlag {
+		fmt.Println(commit)
+		fmt.Println(version)
+		os.Exit(0)
+	}
+
+	var opts []redis.DialOption
+	if redisPassword != "" {
+		opts = append(opts, redis.DialPassword(redisPassword))
+	}
+	pool := redis.Pool{
+		Dial: func() (redis.Conn, error) {
+			return redis.DialURL(redisURL, opts...)
+		},
+		MaxIdle:   10,
+		MaxActive: 20,
+		TestOnBorrow: func(c redis.Conn, t time.Time) error {
+			if time.Since(t) > 10*time.Second {
+				_, err := c.Do("PING")
+				return err
+			}
+			return nil
+		},
+		Wait: true,
+	}
+
+	run(&pool, graphql, time.Duration(interval)*time.Minute)
+
+}
+
+func run(pool *redis.Pool, graphql string, interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	log.Println("warmer started")
+	for {
+		select {
+		case <-ticker.C:
+			err := warmTwins(pool, graphql)
+			if err != nil {
+				log.Printf("failed to warm twins: %s", err.Error())
+			}
+		}
+	}
+}