Merge branch 'master' into feat/autoshard-store-api

ci/Jenkinsfile.docker

@@ -17,7 +17,7 @@ pipeline {
     string(
       name: 'IMAGE_TAG',
       description: 'Docker image tag.',
-      defaultValue: getDefaultImageTag()
+      defaultValue: getDefaultImageTag(params.IMAGE_TAG)
     )
     string(
       name: 'DOCKER_CRED',
@@ -109,10 +109,11 @@ def discordNotify(Map args=[:]) {
   }
 }
 
-def getDefaultImageTag() {
+def getDefaultImageTag(currentValue) {
   switch (env.JOB_BASE_NAME) {
     case 'docker-latest':  return 'latest'
     case 'docker-release': return 'stable'
-    default:               return ''
+    case 'docker-manual':  return ''
+    default:               return currentValue
   }
 }

cmd/waku/flags.go

@@ -330,12 +330,6 @@ var (
 		Destination: &options.Store.DatabaseURL,
 		EnvVars:     []string{"WAKUNODE2_STORE_MESSAGE_DB_URL"},
 	})
-	StoreMessageDBVacuum = altsrc.NewBoolFlag(&cli.BoolFlag{
-		Name:        "store-message-db-vacuum",
-		Usage:       "Enable database vacuuming at start.",
-		Destination: &options.Store.Vacuum,
-		EnvVars:     []string{"WAKUNODE2_STORE_MESSAGE_DB_VACUUM"},
-	})
 	StoreMessageDBMigration = altsrc.NewBoolFlag(&cli.BoolFlag{
 		Name:        "store-message-db-migration",
 		Usage:       "Enable database migration at start.",

cmd/waku/main.go

@@ -65,7 +65,6 @@ func main() {
 		StoreMessageDBURL,
 		StoreMessageRetentionTime,
 		StoreMessageRetentionCapacity,
-		StoreMessageDBVacuum,
 		StoreMessageDBMigration,
 		FilterFlag,
 		FilterNode,

cmd/waku/node.go

@@ -110,9 +110,7 @@ func Execute(options NodeOptions) error {
 	var db *sql.DB
 	var migrationFn func(*sql.DB) error
 	if requiresDB(options) && options.Store.Migration {
-		dbSettings := dbutils.DBSettings{
-			Vacuum: options.Store.Vacuum,
-		}
+		dbSettings := dbutils.DBSettings{}
 		db, migrationFn, err = dbutils.ExtractDBAndMigration(options.Store.DatabaseURL, dbSettings, logger)
 		if err != nil {
 			return nonRecoverErrorMsg("could not connect to DB: %w", err)

cmd/waku/options.go

@@ -81,7 +81,6 @@ type StoreOptions struct {
 	RetentionMaxMessages int
 	//ResumeNodes          []multiaddr.Multiaddr
 	Nodes     []multiaddr.Multiaddr
-	Vacuum    bool
 	Migration bool
 }
 

cmd/waku/server/rest/utils_test.go

@@ -13,7 +13,7 @@ import (
 
 func MemoryDB(t *testing.T) *persistence.DBStore {
 	var db *sql.DB
-	db, err := sqlite.NewDB(":memory:", false, utils.Logger())
+	db, err := sqlite.NewDB(":memory:", utils.Logger())
 	require.NoError(t, err)
 
 	dbStore, err := persistence.NewDBStore(prometheus.DefaultRegisterer, utils.Logger(), persistence.WithDB(db), persistence.WithMigrations(sqlite.Migrations))

library/node.go

@@ -128,7 +128,7 @@ func NewNode(configJSON string) error {
 	if *config.EnableStore {
 		var db *sql.DB
 		var migrationFn func(*sql.DB) error
-		db, migrationFn, err = dbutils.ExtractDBAndMigration(*config.DatabaseURL, dbutils.DBSettings{Vacuum: true}, utils.Logger())
+		db, migrationFn, err = dbutils.ExtractDBAndMigration(*config.DatabaseURL, dbutils.DBSettings{}, utils.Logger())
 		if err != nil {
 			return err
 		}

tests/string_generators_test.go

@@ -0,0 +1,46 @@
+package tests
+
+import (
+	"github.com/stretchr/testify/require"
+	"github.com/waku-org/go-waku/waku/v2/utils"
+	"go.uber.org/zap"
+	"strconv"
+	"testing"
+)
+
+func TestStringGenerators(t *testing.T) {
+
+	log := utils.Logger()
+
+	// Generate string and print out to console
+	for i := 0; i < 1000; i++ {
+		x, err := GenerateRandomASCIIString(1, 4097)
+		require.NoError(t, err)
+		log.Info("Generated random ASCII string", zap.String(strconv.Itoa(i), x))
+
+		x, err = GenerateRandomUTF8String(1, 4097, false)
+		require.NoError(t, err)
+		log.Info("Generated random UTF8 string", zap.String(strconv.Itoa(i), x))
+
+		x, err = GenerateRandomUTF8String(1, 4097, true)
+		require.NoError(t, err)
+		log.Info("Generated uncommon UTF8 string", zap.String(strconv.Itoa(i), x))
+
+		x, err = GenerateRandomJSONString()
+		require.NoError(t, err)
+		log.Info("Generated random JSON string", zap.String(strconv.Itoa(i), x))
+
+		x, err = GenerateRandomBase64String(1025)
+		require.NoError(t, err)
+		log.Info("Generated random Base64 string", zap.String(strconv.Itoa(i), x))
+
+		x, err = GenerateRandomURLEncodedString(2049)
+		require.NoError(t, err)
+		log.Info("Generated random URL encoded string", zap.String(strconv.Itoa(i), x))
+
+		x, err = GenerateRandomSQLInsert()
+		require.NoError(t, err)
+		log.Info("Generated random SQL insert string", zap.String(strconv.Itoa(i), x))
+	}
+
+}

tests/utils.go

@@ -1,16 +1,23 @@
 package tests
 
 import (
+	"bytes"
 	"context"
 	"crypto/ecdsa"
 	"crypto/rand"
+	"encoding/base64"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"math"
+	"math/big"
 	"net"
+	"net/url"
 	"strconv"
+	"strings"
 	"testing"
+	"unicode/utf8"
 
 	gcrypto "github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
@@ -220,3 +227,165 @@ func ExtractIP(addr multiaddr.Multiaddr) (*net.TCPAddr, error) {
 		Port: port,
 	}, nil
 }
+
+func RandomInt(min, max int) (int, error) {
+	n, err := rand.Int(rand.Reader, big.NewInt(int64(max-min+1)))
+	if err != nil {
+		return 0, err
+	}
+	return min + int(n.Int64()), nil
+}
+
+func RandomBytes(n int) ([]byte, error) {
+	b := make([]byte, n)
+	_, err := rand.Read(b)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}
+
+func GenerateRandomASCIIString(minLength int, maxLength int) (string, error) {
+	length, err := rand.Int(rand.Reader, big.NewInt(int64(maxLength-minLength+1)))
+	if err != nil {
+		return "", err
+	}
+	length.SetInt64(length.Int64() + int64(minLength))
+
+	const chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+	result := make([]byte, length.Int64())
+	for i := range result {
+		num, err := rand.Int(rand.Reader, big.NewInt(int64(len(chars))))
+		if err != nil {
+			return "", err
+		}
+		result[i] = chars[num.Int64()]
+	}
+
+	return string(result), nil
+}
+
+func GenerateRandomUTF8String(minLength int, maxLength int, withUncommon bool) (string, error) {
+	length, err := rand.Int(rand.Reader, big.NewInt(int64(maxLength-minLength+1)))
+	if err != nil {
+		return "", err
+	}
+	length.SetInt64(length.Int64() + int64(minLength))
+
+	var (
+		runes      []rune
+		start, end int
+	)
+
+	if withUncommon {
+		// Unicode range for uncommon or unprintable characters, the Private Use Area (E000–F8FF)
+		start = 0xE000
+		end = 0xF8FF
+	} else {
+		// Define unicode range
+		start = 0x0020 // Space character
+		end = 0x007F   // Tilde (~)
+	}
+
+	for i := 0; int64(i) < length.Int64(); i++ {
+		randNum, err := rand.Int(rand.Reader, big.NewInt(int64(end-start+1)))
+		if err != nil {
+			return "", err
+		}
+		char := rune(start + int(randNum.Int64()))
+		if !utf8.ValidRune(char) {
+			continue
+		}
+		runes = append(runes, char)
+	}
+
+	return string(runes), nil
+}
+
+func GenerateRandomJSONString() (string, error) {
+	// With 5 key-value pairs
+	m := make(map[string]interface{})
+	for i := 0; i < 5; i++ {
+		key, err := GenerateRandomASCIIString(1, 20)
+		if err != nil {
+			return "", err
+		}
+		value, err := GenerateRandomASCIIString(1, 4097)
+		if err != nil {
+			return "", err
+		}
+
+		m[key] = value
+	}
+
+	// Marshal the map into a JSON string
+	var buf bytes.Buffer
+	encoder := json.NewEncoder(&buf)
+	encoder.SetEscapeHTML(false)
+	err := encoder.Encode(m)
+	if err != nil {
+		return "", err
+	}
+
+	return buf.String(), nil
+}
+
+func GenerateRandomBase64String(length int) (string, error) {
+	bytes, err := RandomBytes(length)
+	if err != nil {
+		return "", err
+	}
+
+	return base64.StdEncoding.EncodeToString(bytes), nil
+}
+
+func GenerateRandomURLEncodedString(length int) (string, error) {
+	randomString, err := GenerateRandomASCIIString(1, 4097)
+	if err != nil {
+		return "", err
+	}
+
+	// URL-encode the random string
+	return url.QueryEscape(randomString), nil
+}
+
+func GenerateRandomSQLInsert() (string, error) {
+	// Random table name
+	tableName, err := GenerateRandomASCIIString(1, 10)
+	if err != nil {
+		return "", err
+	}
+
+	// Random column names
+	columnCount, err := RandomInt(3, 6)
+	if err != nil {
+		return "", err
+	}
+	columnNames := make([]string, columnCount)
+	for i := 0; i < columnCount; i++ {
+		columnName, err := GenerateRandomASCIIString(1, 20)
+		if err != nil {
+			return "", err
+		}
+		columnNames[i] = columnName
+	}
+
+	// Random values
+	values := make([]string, columnCount)
+	for i := 0; i < columnCount; i++ {
+		value, err := GenerateRandomASCIIString(1, 100)
+		if err != nil {
+			return "", err
+		}
+		values[i] = "'" + value + "'"
+	}
+
+	query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);",
+		tableName,
+		strings.Join(columnNames, ", "),
+		strings.Join(values, ", "))
+
+	return query, nil
+}

waku/persistence/postgres/postgres.go

@@ -1,10 +1,8 @@
 package postgres
 
 import (
-	"context"
 	"database/sql"
 	"fmt"
-	"time"
 
 	"github.com/golang-migrate/migrate/v4/database"
 	"github.com/golang-migrate/migrate/v4/database/pgx"
@@ -15,55 +13,13 @@ import (
 	"go.uber.org/zap"
 )
 
-func executeVacuum(db *sql.DB, logger *zap.Logger) error {
-	logger.Info("starting PostgreSQL database vacuuming")
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	errCh := make(chan error)
-
-	go func() {
-		defer cancel()
-		_, err := db.Exec("VACUUM FULL")
-		if err != nil {
-			errCh <- err
-		}
-	}()
-
-	t := time.NewTicker(2 * time.Minute)
-	defer t.Stop()
-
-loop:
-	for {
-		select {
-		case <-ctx.Done():
-			break loop
-		case err := <-errCh:
-			return err
-		case <-t.C:
-			logger.Info("still vacuuming...")
-		}
-	}
-
-	logger.Info("finished PostgreSQL database vacuuming")
-	return nil
-}
-
 // NewDB connects to postgres DB in the specified path
-func NewDB(dburl string, shouldVacuum bool, logger *zap.Logger) (*sql.DB, error) {
+func NewDB(dburl string, logger *zap.Logger) (*sql.DB, error) {
 	db, err := sql.Open("pgx", dburl)
 	if err != nil {
 		return nil, err
 	}
 
-	if shouldVacuum {
-		err := executeVacuum(db, logger)
-		if err != nil {
-			return nil, err
-		}
-	}
-
 	return db, nil
 }