Allow redis database selection for all Server and Configurator redis …

…connection pools (not in Redis Source yet)
jitsucom · Aug 10, 2022 · d189010 · d189010
2 parents 24424e4 + 058bd11
commit d189010
Show file tree

Hide file tree

Showing 10 changed files with 34 additions and 22 deletions.
diff --git a/configurator/backend/main.go b/configurator/backend/main.go
@@ -343,8 +343,10 @@ func newAuthorizator(ctx context.Context, vp *viper.Viper, mailSender authorizat
 		port := vp.GetInt("auth.redis.port")
 		sentinelMaster := vp.GetString("auth.redis.sentinel_master_name")
 		redisPassword := vp.GetString("auth.redis.password")
+		redisDatabase := vp.GetInt("auth.redis.database")
+
 		tlsSkipVerify := vp.GetBool("auth.redis.tls_skip_verify")
-		redisPoolFactory := meta.NewRedisPoolFactory(host, port, redisPassword, tlsSkipVerify, sentinelMaster)
+		redisPoolFactory := meta.NewRedisPoolFactory(host, port, redisPassword, redisDatabase, tlsSkipVerify, sentinelMaster)
 		if defaultPort, ok := redisPoolFactory.CheckAndSetDefaultPort(); ok {
 			logging.Infof("auth.redis.port isn't configured. Will be used default: %d", defaultPort)
 		}
@@ -464,10 +466,12 @@ func initializeStorage(vp *viper.Viper) (storages.ConfigurationsStorage, *meta.R
 
 		port := vp.GetInt("storage.redis.port")
 		password := vp.GetString("storage.redis.password")
+		database := vp.GetInt("storage.redis.database")
+
 		tlsSkipVerify := vp.GetBool("storage.redis.tls_skip_verify")
 		sentinelMaster := vp.GetString("storage.redis.sentinel_master_name")
 
-		redisPoolFactory := meta.NewRedisPoolFactory(host, port, password, tlsSkipVerify, sentinelMaster)
+		redisPoolFactory := meta.NewRedisPoolFactory(host, port, password, database, tlsSkipVerify, sentinelMaster)
 		if defaultPort, ok := redisPoolFactory.CheckAndSetDefaultPort(); ok {
 			logging.Infof("storage.redis.port isn't configured. Will be used default: %d", defaultPort)
 		}

diff --git a/documentation/configuration/index.mdx b/documentation/configuration/index.mdx
@@ -97,6 +97,7 @@ meta:
       host: redis_host
       port: 6379
       password: secret_password
+      database: 0
 
 node:
   pool_size: 1 # default

diff --git a/documentation/deployment/scale.mdx b/documentation/deployment/scale.mdx
@@ -80,6 +80,7 @@ coordination:
     host: your_redis_host
     port: 6379
     password: secret_password
+    database: 0
 
 ```
 
@@ -98,6 +99,7 @@ meta.storage:
     host: your_redis_host
     port: 6379
     password: secret_password
+    database: 0
 
 coordination:
   type: redis
@@ -117,6 +119,7 @@ meta:
       host: your_sentinel_host # eg: localhost, redis-headless.redis.svc.cluster.local
       port: 26379
       sentinel_master_name: redis_dev_master
+      database: 0
 
 ## or configuration via sentinel URL
 meta:
@@ -131,12 +134,14 @@ storage:
     sentinel_master_name: 'redis_dev_master'
     port: 26379
     tls_skip_verify: '${env.TLS_SKIP_VERIFY|false}'
+    database: 0
 
 auth:
   redis:
     host: '${env.REDIS_URL}'
     port: 26379
     sentinel_master_name: 'redis_dev_master'
+    database: 0
 ```
 
 Every **Jitsu Server** instance with configured coordination sends heartbeat requests every 90 seconds.

diff --git a/release-tool.sh b/release-tool.sh
@@ -24,6 +24,7 @@ function fail() {
 
 function build_configurator() {
   echo "Building Configurator UI locally.."
+  rm -f configurator/frontend/main/public/app-version.json
   rm -f configurator/backend/build/dist/configurator && rm -rf configurator/frontend/main/build && \
   cd configurator/frontend/ && pnpm clean && pnpm i && CI=false ANALYTICS_KEYS='{"eventnative": "js.gpon6lmpwquappfl07tuq.ka5sxhsm08cmblny72tevi", "sentry": "https://[email protected]/6365760"}' pnpm build && \
   cd ../../ || fail 'Configurator build failed'

diff --git a/server/drivers/redis/redis.go b/server/drivers/redis/redis.go
@@ -62,7 +62,7 @@ func NewRedis(_ context.Context, sourceConfig *base.SourceConfig, collection *ba
 		return nil, fmt.Errorf("Error casting redis port [%s] to int: %v", config.Port.String(), err)
 	}
 
-	factory := meta.NewRedisPoolFactory(config.Host, int(intPort), config.Password, config.TLSSkipVerify, config.SentinelMasterName)
+	factory := meta.NewRedisPoolFactory(config.Host, int(intPort), config.Password, 0, config.TLSSkipVerify, config.SentinelMasterName)
 	if defaultPort, ok := factory.CheckAndSetDefaultPort(); ok {
 		logging.Warnf("[%s] port wasn't provided. Will be used default one: %d", sourceConfig.SourceID, defaultPort)
 	}
@@ -96,7 +96,7 @@ func TestRedis(sourceConfig *base.SourceConfig) error {
 		return fmt.Errorf("Error casting redis port [%s] to int: %v", config.Port.String(), err)
 	}
 
-	factory := meta.NewRedisPoolFactory(config.Host, int(intPort), config.Password, config.TLSSkipVerify, config.SentinelMasterName)
+	factory := meta.NewRedisPoolFactory(config.Host, int(intPort), config.Password, 0, config.TLSSkipVerify, config.SentinelMasterName)
 	factory.CheckAndSetDefaultPort()
 
 	pool, err := factory.Create()

diff --git a/server/main.go b/server/main.go
@@ -570,11 +570,7 @@ func initializeCoordinationService(ctx context.Context, metaStorageConfiguration
 		}
 
 		telemetry.Coordination("redis")
-		factory := meta.NewRedisPoolFactory(host,
-			coordinationRedisConfiguration.GetInt("port"),
-			coordinationRedisConfiguration.GetString("password"),
-			coordinationRedisConfiguration.GetBool("tls_skip_verify"),
-			coordinationRedisConfiguration.GetString("sentinel_master_name"))
+		factory := meta.NewRedisPoolFactory(host, coordinationRedisConfiguration.GetInt("port"), coordinationRedisConfiguration.GetString("password"), coordinationRedisConfiguration.GetInt("database"), coordinationRedisConfiguration.GetBool("tls_skip_verify"), coordinationRedisConfiguration.GetString("sentinel_master_name"))
 		factory.CheckAndSetDefaultPort()
 		return coordination.NewRedisService(ctx, appconfig.Instance.ServerName, factory)
 	}
@@ -601,11 +597,7 @@ func initializeEventsQueueFactory(metaStorageConfiguration *viper.Viper) (*event
 	var eventsQueueRedisPool *meta.RedisPool
 	var err error
 	if redisConfigurationSource != nil && redisConfigurationSource.GetString("host") != "" {
-		factory := meta.NewRedisPoolFactory(redisConfigurationSource.GetString("host"),
-			redisConfigurationSource.GetInt("port"),
-			redisConfigurationSource.GetString("password"),
-			redisConfigurationSource.GetBool("tls_skip_verify"),
-			redisConfigurationSource.GetString("sentinel_master_name"))
+		factory := meta.NewRedisPoolFactory(redisConfigurationSource.GetString("host"), redisConfigurationSource.GetInt("port"), redisConfigurationSource.GetString("password"), redisConfigurationSource.GetInt("database"), redisConfigurationSource.GetBool("tls_skip_verify"), redisConfigurationSource.GetString("sentinel_master_name"))
 		opts := meta.DefaultOptions
 		opts.MaxActive = 5000
 		factory.WithOptions(opts)

diff --git a/server/meta/redis_factory.go b/server/meta/redis_factory.go
@@ -57,14 +57,15 @@ type RedisPoolFactory struct {
 	host               string
 	port               int
 	password           string
+	database           int
 	sentinelMasterName string
 	tlsSkipVerify      bool
 
 	options Options
 }
 
 //NewRedisPoolFactory returns filled RedisPoolFactory and removes quotes in host
-func NewRedisPoolFactory(host string, port int, password string, tlsSkipVerify bool, sentinelMasterMame string) *RedisPoolFactory {
+func NewRedisPoolFactory(host string, port int, password string, database int, tlsSkipVerify bool, sentinelMasterMame string) *RedisPoolFactory {
 	host = strings.TrimPrefix(host, `"`)
 	host = strings.TrimPrefix(host, `'`)
 	host = strings.TrimSuffix(host, `"`)
@@ -74,6 +75,7 @@ func NewRedisPoolFactory(host string, port int, password string, tlsSkipVerify b
 		host:               host,
 		port:               port,
 		password:           password,
+		database:           database,
 		tlsSkipVerify:      tlsSkipVerify,
 		sentinelMasterName: sentinelMasterMame,
 		options:            DefaultOptions,
@@ -149,6 +151,9 @@ func (rpf *RedisPoolFactory) getSentinelAndDialFunc() (*sentinel.Sentinel, func(
 	defaultDialWriteTimeout := redis.DialWriteTimeout(rpf.options.DefaultDialWriteTimeout)
 
 	options := []redis.DialOption{defaultDialConnectTimeout, defaultDialReadTimeout, defaultDialWriteTimeout}
+	if rpf.database > 0 {
+		options = append(options, redis.DialDatabase(rpf.database))
+	}
 
 	// 1. redis:// redis://
 	if rpf.isURL() || rpf.isSecuredURL() {

diff --git a/server/meta/redis_factory_test.go b/server/meta/redis_factory_test.go
@@ -81,31 +81,31 @@ func TestGetSentinelAndDialFunc(t *testing.T) {
 	}{
 		{
 			"redis URL",
-			NewRedisPoolFactory("redis://localhost:6379", 0, "", false, ""),
+			NewRedisPoolFactory("redis://localhost:6379", 0, "", 0, false, ""),
 			false,
 			"",
 		},
 		{
 			"redis secured URL",
-			NewRedisPoolFactory("rediss://localhost:6379", 0, "", false, ""),
+			NewRedisPoolFactory("rediss://localhost:6379", 0, "", 0, false, ""),
 			false,
 			"",
 		},
 		{
 			"redis sentinel URL",
-			NewRedisPoolFactory("sentinel://master:pas@localhost:6379", 0, "", false, ""),
+			NewRedisPoolFactory("sentinel://master:pas@localhost:6379", 0, "", 0, false, ""),
 			true,
 			"",
 		},
 		{
 			"redis plain config",
-			NewRedisPoolFactory("host", 0, "pass", false, ""),
+			NewRedisPoolFactory("host", 0, "pass", 0, false, ""),
 			false,
 			"",
 		},
 		{
 			"redis plain config with sentinel",
-			NewRedisPoolFactory("host", 0, "pass", false, "sent"),
+			NewRedisPoolFactory("host", 0, "pass", 0, false, "sent"),
 			true,
 			"",
 		},

diff --git a/server/meta/storage.go b/server/meta/storage.go
@@ -76,9 +76,11 @@ func InitializeStorage(metaStorageConfiguration *viper.Viper) (Storage, error) {
 	host := metaStorageConfiguration.GetString("redis.host")
 	port := metaStorageConfiguration.GetInt("redis.port")
 	password := metaStorageConfiguration.GetString("redis.password")
+	database := metaStorageConfiguration.GetInt("redis.database")
+
 	sentinelMaster := metaStorageConfiguration.GetString("redis.sentinel_master_name")
 	tlsSkipVerify := metaStorageConfiguration.GetBool("redis.tls_skip_verify")
-	factory := NewRedisPoolFactory(host, port, password, tlsSkipVerify, sentinelMaster)
+	factory := NewRedisPoolFactory(host, port, password, database, tlsSkipVerify, sentinelMaster)
 	factory.CheckAndSetDefaultPort()
 
 	logging.Infof("🏪 Initializing meta storage redis [%s]...", factory.Details())

diff --git a/server/users/storage.go b/server/users/storage.go
@@ -59,11 +59,13 @@ func InitializeStorage(enabled bool, metaStorageConfiguration *viper.Viper) (Sto
 	host := redisConfigurationSource.GetString("host")
 	port := redisConfigurationSource.GetInt("port")
 	password := redisConfigurationSource.GetString("password")
+	database := redisConfigurationSource.GetInt("database")
+
 	sentinelMaster := redisConfigurationSource.GetString("sentinel_master_name")
 	tlsSkipVerify := redisConfigurationSource.GetBool("tls_skip_verify")
 	anonymousEventsMinutesTTL := redisConfigurationSource.GetInt("ttl_minutes.anonymous_events")
 
-	factory := meta.NewRedisPoolFactory(host, port, password, tlsSkipVerify, sentinelMaster)
+	factory := meta.NewRedisPoolFactory(host, port, password, database, tlsSkipVerify, sentinelMaster)
 	options := factory.GetOptions()
 	options.MaxActive = 100
 	factory.WithOptions(options)