From e7747e3271cb6b7c7ecb7732ab3d7cbdfdb5e5e5 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 23 Aug 2023 16:18:41 +0300 Subject: [PATCH 01/36] Disable read-only mode for replicas on minion if standby failover is used --- common/templates/redis/redis-6.0.conf | 5 +++++ common/templates/redis/redis-6.2.conf | 5 +++++ common/templates/redis/redis-7.0.conf | 5 +++++ common/templates/redis/redis-7.2.conf | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/common/templates/redis/redis-6.0.conf b/common/templates/redis/redis-6.0.conf index d18f757..c22d02d 100644 --- a/common/templates/redis/redis-6.0.conf +++ b/common/templates/redis/redis-6.0.conf @@ -455,7 +455,12 @@ replica-serve-stale-data yes # such as CONFIG, DEBUG, and so forth. To a limited extent you can improve # security of read only replicas using 'rename-command' to shadow all the # administrative / dangerous commands. + +{{if and .RDS.IsFailoverStandby .RDS.IsMinion}} +replica-read-only no +{{else}} replica-read-only yes +{{end}} # Replication SYNC strategy: disk or socket. # diff --git a/common/templates/redis/redis-6.2.conf b/common/templates/redis/redis-6.2.conf index 20b55b4..da0b79e 100644 --- a/common/templates/redis/redis-6.2.conf +++ b/common/templates/redis/redis-6.2.conf @@ -550,7 +550,12 @@ replica-serve-stale-data yes # such as CONFIG, DEBUG, and so forth. To a limited extent you can improve # security of read only replicas using 'rename-command' to shadow all the # administrative / dangerous commands. + +{{if and .RDS.IsFailoverStandby .RDS.IsMinion}} +replica-read-only no +{{else}} replica-read-only yes +{{end}} # Replication SYNC strategy: disk or socket. # diff --git a/common/templates/redis/redis-7.0.conf b/common/templates/redis/redis-7.0.conf index 2d3e6ec..faad6c7 100644 --- a/common/templates/redis/redis-7.0.conf +++ b/common/templates/redis/redis-7.0.conf @@ -596,7 +596,12 @@ replica-serve-stale-data yes # such as CONFIG, DEBUG, and so forth. To a limited extent you can improve # security of read only replicas using 'rename-command' to shadow all the # administrative / dangerous commands. + +{{if and .RDS.IsFailoverStandby .RDS.IsMinion}} +replica-read-only no +{{else}} replica-read-only yes +{{end}} # Replication SYNC strategy: disk or socket. # diff --git a/common/templates/redis/redis-7.2.conf b/common/templates/redis/redis-7.2.conf index b190199..bc0cc49 100644 --- a/common/templates/redis/redis-7.2.conf +++ b/common/templates/redis/redis-7.2.conf @@ -603,7 +603,12 @@ replica-serve-stale-data yes # such as CONFIG, DEBUG, and so forth. To a limited extent you can improve # security of read only replicas using 'rename-command' to shadow all the # administrative / dangerous commands. + +{{if and .RDS.IsFailoverStandby .RDS.IsMinion}} +replica-read-only no +{{else}} replica-read-only yes +{{end}} # Replication SYNC strategy: disk or socket. # From f95828206c165738383730f96c9c1c1ef56a0da3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Sep 2023 03:10:35 +0000 Subject: [PATCH 02/36] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 6 +++--- .github/workflows/codeql.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb72db9..2f5f9bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v4 @@ -56,7 +56,7 @@ jobs: steps: - name: Code checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Login to GitHub Container Registry uses: docker/login-action@v2 @@ -78,7 +78,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Check spelling continue-on-error: true diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 37b39f4..58ae291 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 2 From 7e46a3e8c7811a4b204d3b765b1f990ff7ae3461 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Fri, 8 Sep 2023 11:46:27 +0300 Subject: [PATCH 03/36] [cli] Improve properties filtering in 'conf' command --- cli/cli.go | 4 ++-- cli/command_conf.go | 32 +++++++++++++++++++++++++------- cli/command_help.go | 5 +++-- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/cli/cli.go b/cli/cli.go index 4ed5b40..aa0ec14 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -43,7 +43,7 @@ import ( const ( APP = "RDS" - VER = "1.1.0" + VER = "1.2.0" DESC = "Tool for Redis orchestration" ) @@ -980,7 +980,7 @@ func genUsage() *usage.Info { info.AddCommand(COMMAND_CLIENTS, "Show list of connected clients", "id", "?filter") info.AddCommand(COMMAND_TRACK, "Show interactive info about Redis instance", "id", "?interval") info.AddCommand(COMMAND_CONF, "Show configuration of Redis instance", "id", "?property") - info.AddCommand(COMMAND_LIST, "Show list of all Redis instances", "?filter") + info.AddCommand(COMMAND_LIST, "Show list of all Redis instances", "?filter…") info.AddCommand(COMMAND_STATS, "Show overall statistics") info.AddCommand(COMMAND_TOP, "Show instances top", "?field", "?num") info.AddCommand(COMMAND_TOP_DIFF, "Compare current and dumped top data", "file", "?field", "?num") diff --git a/cli/command_conf.go b/cli/command_conf.go index db91ec7..db93994 100644 --- a/cli/command_conf.go +++ b/cli/command_conf.go @@ -9,6 +9,7 @@ package cli import ( "fmt" + "strings" "time" "github.com/essentialkaos/ek/v12/fmtutil/table" @@ -91,7 +92,7 @@ func ConfCommand(args CommandArgs) int { // printConfInfo shows difference between file and in-memory config func printConfInfo(fileConfig *REDIS.Config, diff []REDIS.ConfigPropDiff, filter []string) { hasData := false - hasFilter := len(filter) != 0 + t := table.NewTable("NAME", "VALUE") for _, prop := range fileConfig.Props { @@ -99,14 +100,16 @@ func printConfInfo(fileConfig *REDIS.Config, diff []REDIS.ConfigPropDiff, filter continue } - if hasFilter && !sliceutil.Contains(filter, prop) { + propFmt, found := filterConfProp(prop, filter) + + if !found { continue } hasData = true if len(diff) == 0 || prop == "user" { - printConfProps(t, prop, fileConfig.Data[prop], "", true) + printConfProps(t, propFmt, fileConfig.Data[prop], "", true) continue } @@ -114,13 +117,13 @@ func printConfInfo(fileConfig *REDIS.Config, diff []REDIS.ConfigPropDiff, filter switch { case diffInfo.PropName != "": - printConfProps(t, prop, fileConfig.Data[prop], diffInfo.MemValue, false) + printConfProps(t, propFmt, fileConfig.Data[prop], diffInfo.MemValue, false) case prop == "replicaof" && findDiffProp(diff, "slaveof").PropName != "": - printConfProps(t, prop, fileConfig.Data[prop], findDiffProp(diff, "slaveof").MemValue, false) + printConfProps(t, propFmt, fileConfig.Data[prop], findDiffProp(diff, "slaveof").MemValue, false) case prop == "slaveof" && findDiffProp(diff, "replicaof").PropName != "": - printConfProps(t, prop, fileConfig.Data[prop], findDiffProp(diff, "replicaof").MemValue, false) + printConfProps(t, propFmt, fileConfig.Data[prop], findDiffProp(diff, "replicaof").MemValue, false) default: - printConfProps(t, prop, fileConfig.Data[prop], "", true) + printConfProps(t, propFmt, fileConfig.Data[prop], "", true) } } @@ -150,6 +153,21 @@ func printConfProps(t *table.Table, prop string, values []string, curValue strin } } +// filterConfProp filters configuration properties +func filterConfProp(prop string, filter []string) (string, bool) { + if len(filter) == 0 { + return prop, true + } + + for _, f := range filter { + if strings.Contains(prop, f) { + return strings.ReplaceAll(prop, f, "{_}"+f+"{!}"), true + } + } + + return prop, false +} + // findDiffProp tries to find changed properties func findDiffProp(diff []REDIS.ConfigPropDiff, prop string) REDIS.ConfigPropDiff { for _, d := range diff { diff --git a/cli/command_help.go b/cli/command_help.go index 82927b1..dc58607 100644 --- a/cli/command_help.go +++ b/cli/command_help.go @@ -435,13 +435,14 @@ func helpCommandConf() { desc: "Print values from the configuration file and in-memory configuration.", arguments: []helpInfoArgument{ {"id", "Instance unique ID", false}, - {"property", "Configuration property", true}, + {"filter…", "Property name filters", true}, }, options: []helpInfoArgument{ {getNiceOptions(OPT_PRIVATE), "Show private info", false}, }, examples: []helpInfoExample{ - {"", "1", "Show configuration for instance with ID 1"}, + {"", "1", "Show configuration of instance with ID 1"}, + {"", "1 append sync", "Show properties with 'append' or 'sync' in the name of instance with ID 1"}, }, }.render() } From 761df1ff7a6967be18ae45ecd31e3c3c3f86765c Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Fri, 8 Sep 2023 12:07:27 +0300 Subject: [PATCH 04/36] [cli] Improve usage info --- cli/cli.go | 10 +++++----- cli/command_help.go | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cli/cli.go b/cli/cli.go index aa0ec14..4bc7738 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -851,11 +851,11 @@ func showSmartUsage() { info.AddCommand(COMMAND_CLI, "Run CLI connected to Redis instance", "id:db", "?command") info.AddCommand(COMMAND_CPU, "Calculate instance CPU usage", "id", "?period") info.AddCommand(COMMAND_MEMORY, "Show instance memory usage", "id") - info.AddCommand(COMMAND_INFO, "Show system info about Redis instance", "id", "?section") + info.AddCommand(COMMAND_INFO, "Show system info about Redis instance", "id", "?section…") info.AddCommand(COMMAND_CLIENTS, "Show list of connected clients", "id", "?filter") info.AddCommand(COMMAND_TRACK, "Show interactive info about Redis instance", "id", "?interval") - info.AddCommand(COMMAND_CONF, "Show configuration of Redis instance", "id", "?property") - info.AddCommand(COMMAND_LIST, "Show list of all Redis instances", "?filter") + info.AddCommand(COMMAND_CONF, "Show configuration of Redis instance", "id", "?filter…") + info.AddCommand(COMMAND_LIST, "Show list of all Redis instances", "?filter…") info.AddCommand(COMMAND_STATS, "Show overall statistics") info.AddCommand(COMMAND_STATS_COMMAND, "Show statistics based on the command type", "id") info.AddCommand(COMMAND_STATS_LATENCY, "Show latency statistics based on the command type", "id") @@ -973,13 +973,13 @@ func genUsage() *usage.Info { info.AddCommand(COMMAND_CLI, "Run CLI connected to Redis instance", "id:db", "?command") info.AddCommand(COMMAND_CPU, "Calculate instance CPU usage", "id", "?period") info.AddCommand(COMMAND_MEMORY, "Show instance memory usage", "id") - info.AddCommand(COMMAND_INFO, "Show system info about Redis instance", "id", "?section") + info.AddCommand(COMMAND_INFO, "Show system info about Redis instance", "id", "?section…") info.AddCommand(COMMAND_STATS_COMMAND, "Show statistics based on the command type", "id") info.AddCommand(COMMAND_STATS_LATENCY, "Show latency statistics based on the command type", "id") info.AddCommand(COMMAND_STATS_ERROR, "Show error statistics", "id") info.AddCommand(COMMAND_CLIENTS, "Show list of connected clients", "id", "?filter") info.AddCommand(COMMAND_TRACK, "Show interactive info about Redis instance", "id", "?interval") - info.AddCommand(COMMAND_CONF, "Show configuration of Redis instance", "id", "?property") + info.AddCommand(COMMAND_CONF, "Show configuration of Redis instance", "id", "?filter…") info.AddCommand(COMMAND_LIST, "Show list of all Redis instances", "?filter…") info.AddCommand(COMMAND_STATS, "Show overall statistics") info.AddCommand(COMMAND_TOP, "Show instances top", "?field", "?num") diff --git a/cli/command_help.go b/cli/command_help.go index dc58607..dfceb86 100644 --- a/cli/command_help.go +++ b/cli/command_help.go @@ -286,7 +286,7 @@ func helpCommandStatus() { {"id", "Instance unique ID", false}, }, examples: []helpInfoExample{ - {"", "1", "Show status for instance with ID 1"}, + {"", "1", "Show status of instance with ID 1"}, }, }.render() } @@ -353,14 +353,14 @@ func helpCommandInfo() { desc: "Show system info about Redis instance.", arguments: []helpInfoArgument{ {"id", "Instance unique ID", false}, - {"section", "Info section", true}, + {"section…", "Info section", true}, }, options: []helpInfoArgument{ {getNiceOptions(OPT_FORMAT), "Output format (json|text|xml)", false}, }, examples: []helpInfoExample{ {"", "1", "Show basic info about instance with ID 1"}, - {"", "1 memory", "Show info memory section for instance with ID 1"}, + {"", "1 memory cpu", "Show memory and cpu section of instance with ID 1"}, {"", "1 all", "Show all info (including non-default sections) about instance with ID 1"}, }, }.render() @@ -468,7 +468,7 @@ func helpCommandList() { command: COMMAND_LIST, desc: "Show list of all Redis instances.", arguments: []helpInfoArgument{ - {"filter", "Listing filter", true}, + {"filter…", "Listing filters", true}, }, examples: []helpInfoExample{ {"", "", "Show list of all instances"}, @@ -529,7 +529,7 @@ func helpCommandMemory() { {"id", "Instance unique ID", false}, }, examples: []helpInfoExample{ - {"", "1", "Show memory usage for instance with ID 1"}, + {"", "1", "Show memory usage of instance with ID 1"}, }, }.render() } @@ -554,7 +554,7 @@ func helpCommandStatsCommand() { command: COMMAND_STATS_COMMAND, desc: "Show statistics based on the command type.", examples: []helpInfoExample{ - {"", "1", "Show statistics for instance with ID 1"}, + {"", "1", "Show statistics of instance with ID 1"}, }, }.render() } @@ -565,7 +565,7 @@ func helpCommandStatsLatency() { command: COMMAND_STATS_LATENCY, desc: "Show latency statistics based on the command type.", examples: []helpInfoExample{ - {"", "1", "Show statistics for instance with ID 1"}, + {"", "1", "Show statistics of instance with ID 1"}, }, }.render() } @@ -576,7 +576,7 @@ func helpCommandStatsError() { command: COMMAND_STATS_ERROR, desc: "Show error statistics.", examples: []helpInfoExample{ - {"", "1", "Show statistics for instance with ID 1"}, + {"", "1", "Show statistics of instance with ID 1"}, }, }.render() } From c65769aa30d73e9cffdfa74e6cd48d45c6376b53 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Fri, 8 Sep 2023 12:21:02 +0300 Subject: [PATCH 05/36] [cli] Add -R/--raw option --- cli/cli.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cli/cli.go b/cli/cli.go index 4bc7738..ad5b732 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -71,6 +71,7 @@ const ( OPT_DISABLE_SAVES = "ds:disable-saves" OPT_YES = "y:yes" OPT_SIMPLE = "S:simple" + OPT_RAW = "R:raw" OPT_NO_COLOR = "nc:no-color" OPT_HELP = "h:help" OPT_VERSION = "v:version" @@ -179,6 +180,7 @@ var optMap = options.Map{ OPT_DISABLE_SAVES: {Type: options.BOOL}, OPT_NO_COLOR: {Type: options.BOOL}, OPT_SIMPLE: {Type: options.BOOL}, + OPT_RAW: {Type: options.BOOL}, OPT_YES: {Type: options.BOOL}, OPT_HELP: {Type: options.BOOL}, OPT_VERSION: {Type: options.MIXED}, @@ -343,6 +345,10 @@ func configureUI() { if options.GetB(OPT_YES) { terminal.AlwaysYes = true } + + if options.GetB(OPT_RAW) { + useRawOutput = true + } } // parseOptions parse command line options @@ -945,6 +951,7 @@ func showSmartUsage() { info.AddOption(OPT_FORMAT, "Output format {s-}(text/json/xml){!}", "format") info.AddOption(OPT_YES, "Automatically answer yes for all questions") info.AddOption(OPT_SIMPLE, "Simplify output {s-}(useful for copy-paste){!}") + info.AddOption(OPT_RAW, "Force raw output {s-}(useful for scripts){!}") info.AddOption(OPT_NO_COLOR, "Disable colors in output") info.AddOption(OPT_HELP, "Show this help message") info.AddOption(OPT_VERSION, "Show information about version") @@ -1030,6 +1037,7 @@ func genUsage() *usage.Info { info.AddOption(OPT_FORMAT, "Output format {s-}(text/json/xml){!}", "format") info.AddOption(OPT_YES, "Automatically answer yes for all questions") info.AddOption(OPT_SIMPLE, "Simplify output {s-}(useful for copy-paste){!}") + info.AddOption(OPT_RAW, "Force raw output {s-}(useful for scripts){!}") info.AddOption(OPT_NO_COLOR, "Disable colors in output") info.AddOption(OPT_HELP, "Show this help message") info.AddOption(OPT_VERSION, "Show information about version") From a44dff1fefbdf5947ec8c69a9b16f7581fd323d6 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Fri, 8 Sep 2023 14:16:02 +0300 Subject: [PATCH 06/36] [cli] [core] Add templates validation --- cli/cli.go | 27 +++++--- cli/command_help.go | 12 ++++ cli/common.go | 7 -- core/core.go | 163 +++++++++++++++++++++++++++----------------- 4 files changed, 132 insertions(+), 77 deletions(-) diff --git a/cli/cli.go b/cli/cli.go index ad5b732..9cf6a75 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -145,6 +145,7 @@ const ( COMMAND_TOP_DIFF = "top-diff" COMMAND_TOP_DUMP = "top-dump" COMMAND_TRACK = "track" + COMMAND_VALIDATE_TEMPLATES = "validate-templates" ) const ( @@ -169,6 +170,13 @@ const ( AUTH_SUPERUSER ) +type CommandRoutine struct { + Handler CommandHandler + Auth AuthType + RequireStrictAuth bool + PrettyOutput bool +} + // ////////////////////////////////////////////////////////////////////////////////// // // optMap is map with options data @@ -473,7 +481,7 @@ func initCommands() { } } else { commands[COMMAND_CONF] = &CommandRoutine{ConfCommand, AUTH_NO, false, true} - commands[COMMAND_CLI] = &CommandRoutine{CliCommand, AUTH_NO, true, useRawOutput == false} + commands[COMMAND_CLI] = &CommandRoutine{CliCommand, AUTH_NO, false, useRawOutput == false} commands[COMMAND_SETTINGS] = &CommandRoutine{SettingsCommand, AUTH_NO, false, true} } @@ -512,13 +520,13 @@ func initCommands() { } } - commands[COMMAND_SENTINEL_STATUS] = &CommandRoutine{SentinelStatusCommand, AUTH_NO, true, true} + commands[COMMAND_SENTINEL_STATUS] = &CommandRoutine{SentinelStatusCommand, AUTH_NO, false, true} if CORE.IsSentinelActive() { - commands[COMMAND_SENTINEL_CHECK] = &CommandRoutine{SentinelCheckCommand, AUTH_NO, true, true} - commands[COMMAND_SENTINEL_INFO] = &CommandRoutine{SentinelInfoCommand, AUTH_NO, true, true} - commands[COMMAND_SENTINEL_MASTER] = &CommandRoutine{SentinelMasterCommand, AUTH_NO, true, true} - commands[COMMAND_SENTINEL_RESET] = &CommandRoutine{SentinelResetCommand, AUTH_SUPERUSER, true, true} + commands[COMMAND_SENTINEL_CHECK] = &CommandRoutine{SentinelCheckCommand, AUTH_NO, false, true} + commands[COMMAND_SENTINEL_INFO] = &CommandRoutine{SentinelInfoCommand, AUTH_NO, false, true} + commands[COMMAND_SENTINEL_MASTER] = &CommandRoutine{SentinelMasterCommand, AUTH_NO, false, true} + commands[COMMAND_SENTINEL_RESET] = &CommandRoutine{SentinelResetCommand, AUTH_SUPERUSER, false, true} } } @@ -540,7 +548,8 @@ func initCommands() { commands[COMMAND_RESTART_ALL_PROP] = &CommandRoutine{RestartAllPropCommand, AUTH_SUPERUSER, true, true} } - commands[COMMAND_GEN_TOKEN] = &CommandRoutine{GenTokenCommand, AUTH_NO, true, useRawOutput == false} + commands[COMMAND_VALIDATE_TEMPLATES] = &CommandRoutine{ValidateTemplatesCommand, AUTH_NO, false, true} + commands[COMMAND_GEN_TOKEN] = &CommandRoutine{GenTokenCommand, AUTH_NO, false, useRawOutput == false} commands[COMMAND_HELP] = &CommandRoutine{HelpCommand, AUTH_NO, false, true} for a, c := range aliases { @@ -728,7 +737,7 @@ func getSpellcheckModel() *spellcheck.Model { COMMAND_STATS_COMMAND, COMMAND_STATS_LATENCY, COMMAND_STATS_ERROR, COMMAND_STATUS, COMMAND_STOP, COMMAND_STOP_ALL, COMMAND_STOP_ALL_PROP, COMMAND_STOP_PROP, COMMAND_TAG_ADD, COMMAND_TAG_REMOVE, COMMAND_TOP, - COMMAND_TOP_DIFF, COMMAND_TOP_DUMP, COMMAND_TRACK, + COMMAND_TOP_DIFF, COMMAND_TOP_DUMP, COMMAND_TRACK, COMMAND_VALIDATE_TEMPLATES, }) } @@ -940,6 +949,7 @@ func showSmartUsage() { info.AddCommand(COMMAND_HELP, "Show command usage info", "command") info.AddCommand(COMMAND_SETTINGS, "Show settings from global configuration file", "?section…") info.AddCommand(COMMAND_GEN_TOKEN, "Generate authentication token for sync daemon") + info.AddCommand(COMMAND_VALIDATE_TEMPLATES, "Validate Redis and Sentinel templates") if isMaster { info.AddOption(OPT_SECURE, "Create secure Redis instance with auth support ({y}create{!})") @@ -1029,6 +1039,7 @@ func genUsage() *usage.Info { info.AddCommand(COMMAND_HELP, "Show command usage info", "command") info.AddCommand(COMMAND_SETTINGS, "Show settings from global configuration file", "?section…") info.AddCommand(COMMAND_GEN_TOKEN, "Generate authentication token for sync daemon") + info.AddCommand(COMMAND_VALIDATE_TEMPLATES, "Validate Redis and Sentinel templates") info.AddOption(OPT_SECURE, "Create secure Redis instance with auth support") info.AddOption(OPT_DISABLE_SAVES, "Disable saves for created instance") diff --git a/cli/command_help.go b/cli/command_help.go index dfceb86..bde07ea 100644 --- a/cli/command_help.go +++ b/cli/command_help.go @@ -108,6 +108,7 @@ func HelpCommand(args CommandArgs) int { COMMAND_TOP_DIFF: helpCommandTopDiff, COMMAND_TOP_DUMP: helpCommandTopDump, COMMAND_TRACK: helpCommandTrack, + COMMAND_VALIDATE_TEMPLATES: helpCommandValidateTemplates, } helpFunc, hasInfo := commandList[commandName] @@ -918,6 +919,17 @@ func helpCommandGenToken() { }.render() } +// helpCommandValidateTemplates prints info about "validate-templates" command usage +func helpCommandValidateTemplates() { + helpInfo{ + command: COMMAND_VALIDATE_TEMPLATES, + desc: "Validate Redis and Sentinel configuration file templates.", + examples: []helpInfoExample{ + {"", "", "Validate templates"}, + }, + }.render() +} + // ////////////////////////////////////////////////////////////////////////////////// // // getNiceOptions parse option and return formatted string diff --git a/cli/common.go b/cli/common.go index 2431411..cd6b7c1 100644 --- a/cli/common.go +++ b/cli/common.go @@ -33,13 +33,6 @@ type CommandArgs []string type CommandHandler func(CommandArgs) int -type CommandRoutine struct { - Handler CommandHandler - Auth AuthType - RequireStrictAuth bool - PrettyOutput bool -} - // ////////////////////////////////////////////////////////////////////////////////// // var userExistenceCache map[string]bool diff --git a/core/core.go b/core/core.go index 509e188..4cf379b 100644 --- a/core/core.go +++ b/core/core.go @@ -216,6 +216,13 @@ const ( REPL_TYPE_STANDBY ReplicationType = "standby" ) +type TemplateSource string + +const ( + TEMPLATE_SOURCE_REDIS TemplateSource = "redis.conf" + TEMPLATE_SOURCE_SENTINEL TemplateSource = "sentinel.conf" +) + // ////////////////////////////////////////////////////////////////////////////////// // type SystemStatus struct { @@ -375,6 +382,12 @@ type instanceConfigRDSData struct { IsFailoverSentinel bool } +type sentinelConfigData struct { + Port string + PidFile string + LogFile string +} + // ////////////////////////////////////////////////////////////////////////////////// // var ( @@ -408,6 +421,7 @@ var ( ErrCantReadDaemonizeOption = errors.New("Can't read 'daemonize' option value from instance configuration file") ErrCantDaemonizeInstance = errors.New("Impossible to run instance - 'daemonize' property set to 'no' in configuration file") ErrUnknownReplicationType = errors.New("Unsupported replication type") + ErrUnknownTemplateSource = errors.New("Unknown template source") ) // ////////////////////////////////////////////////////////////////////////////////// // @@ -617,6 +631,33 @@ func ReadSUAuth() (*SuperuserAuth, error) { return auth, err } +// ValidateTemplates validates templates for Redis and Sentinel +func ValidateTemplates() []error { + var errs errutil.Errors + + meta, err := NewInstanceMeta("test", "test") + + if err != nil { + errs.Add(fmt.Errorf("Can't generate instance meta for validation: %w", err)) + } else { + _, err = generateConfigFromTemplate( + TEMPLATE_SOURCE_REDIS, + createConfigFromMeta(meta), + ) + + errs.Add(err) + } + + _, err = generateConfigFromTemplate( + TEMPLATE_SOURCE_SENTINEL, + &sentinelConfigData{}, + ) + + errs.Add(err) + + return errs.All() +} + // HasInstances returns true if that at least one instance exists func HasInstances() bool { return !fsutil.IsEmptyDir(Config.GetS(PATH_META_DIR)) @@ -652,6 +693,11 @@ func GetInstancePIDFilePath(id int) string { return path.Join(Config.GetS(PATH_PID_DIR), strconv.Itoa(id)+".pid") } +// GetStatesFilePath returns path to global states file +func GetStatesFilePath() string { + return path.Join(Config.GetS(MAIN_DIR), STATES_DATA_FILE) +} + // GetInstancePort returns port used by redis for given instance func GetInstancePort(id int) int { return Config.GetI(REDIS_START_PORT) + id @@ -1984,11 +2030,6 @@ func ParseTag(tag string) (string, string) { return tag, "" } -// GetStatesFilePath returns path to global states file -func GetStatesFilePath() string { - return path.Join(Config.GetS(MAIN_DIR), STATES_DATA_FILE) -} - // IsSyncDaemonActive returns true if sync daemon is works func IsSyncDaemonActive() bool { isWorks, _ := initsystem.IsWorks("rds-sync.service") @@ -2617,36 +2658,21 @@ func saveInstanceMeta(meta *InstanceMeta) error { // createInstanceConfig create redis config from template func createInstanceConfig(meta *InstanceMeta) error { - templateData, err := getConfigTemplateData(false) - - if err != nil { - return err - } - - t, err := template.New("redis.conf").Parse(templateData) - - if err != nil { - return err - } - - var bf bytes.Buffer - - ct := template.Must(t, nil) cfg := createConfigFromMeta(meta) - err = ct.Execute(&bf, cfg) + confData, err := generateConfigFromTemplate(TEMPLATE_SOURCE_REDIS, cfg) if err != nil { return err } - err = os.WriteFile(GetInstanceConfigFilePath(meta.ID), bf.Bytes(), 0640) + err = os.WriteFile(GetInstanceConfigFilePath(meta.ID), confData, 0640) if err != nil { return err } hasher := sha256.New() - hasher.Write(bf.Bytes()) + hasher.Write(confData) meta.Config.Hash = fmt.Sprintf("%064x", hasher.Sum(nil)) meta.Config.Date = time.Now().Unix() @@ -2926,45 +2952,27 @@ func generateSentinelConfig() error { return err } - templateData, err := getConfigTemplateData(true) - - if err != nil { - return err - } - - t, err := template.New("sentinel.conf").Parse(templateData) - - if err != nil { - return err - } - - var bf bytes.Buffer - - sentinelProps := struct { - Port string - PidFile string - LogFile string - }{ - Port: sentinelPort, - PidFile: sentinelPidFile, - LogFile: sentinelLogFile, - } - - ct := template.Must(t, nil) - err = ct.Execute(&bf, &sentinelProps) + confData, err := generateConfigFromTemplate( + TEMPLATE_SOURCE_SENTINEL, + &sentinelConfigData{ + Port: sentinelPort, + PidFile: sentinelPidFile, + LogFile: sentinelLogFile, + }, + ) if err != nil { return err } - err = os.WriteFile(sentinelConfig, bf.Bytes(), 0640) + err = os.WriteFile(sentinelConfig, confData, 0640) if err != nil { return err } if !fsutil.IsExist(sentinelLogFile) { - err = os.WriteFile(sentinelLogFile, []byte(""), 0640) + err = fsutil.TouchFile(sentinelLogFile, 0640) if err != nil { return err @@ -2981,35 +2989,66 @@ func generateSentinelConfig() error { // getConfigTemplateData reads configuration data from template // for currently installed Redis/Sentinel version -func getConfigTemplateData(sentinel bool) (string, error) { +func getConfigTemplateData(source TemplateSource) (string, string, error) { var err error - var templateFile string + var templateFile, templateFilePath string currentRedisVer, err := GetRedisVersion() if err != nil { - return "", fmt.Errorf("Can't get Redis version: %w", err) + return "", "", fmt.Errorf("Can't get Redis version: %w", err) } majorRedisVer := fmt.Sprintf("%d.%d", currentRedisVer.Major(), currentRedisVer.Minor()) - if sentinel { - templateFile, err = path.JoinSecure(Config.GetS(TEMPLATES_SENTINEL), "sentinel-"+majorRedisVer+".conf") - } else { - templateFile, err = path.JoinSecure(Config.GetS(TEMPLATES_REDIS), "redis-"+majorRedisVer+".conf") + switch source { + case TEMPLATE_SOURCE_REDIS: + templateFile = "redis-" + majorRedisVer + ".conf" + templateFilePath, err = path.JoinSecure(Config.GetS(TEMPLATES_REDIS), templateFile) + case TEMPLATE_SOURCE_SENTINEL: + templateFile = "sentinel-" + majorRedisVer + ".conf" + templateFilePath, err = path.JoinSecure(Config.GetS(TEMPLATES_SENTINEL), templateFile) + default: + return "", "", ErrUnknownTemplateSource } if err != nil { - return "", fmt.Errorf("Can't create path to configuration template: %w", err) + return "", "", fmt.Errorf("Can't create path to configuration template: %w", err) + } + + data, err := os.ReadFile(templateFilePath) + + if err != nil { + return "", "", fmt.Errorf("Can't read configuration template data: %w", err) } - data, err := os.ReadFile(templateFile) + return templateFile, string(data), nil +} + +// generateConfigFromTemplate generates configuration from template +func generateConfigFromTemplate(source TemplateSource, data any) ([]byte, error) { + templateFile, templateData, err := getConfigTemplateData(source) + + if err != nil { + return nil, err + } + + t, err := template.New(templateFile).Parse(templateData) + + if err != nil { + return nil, fmt.Errorf("Can't parse template data: %w", err) + } + + var bf bytes.Buffer + + ct := template.Must(t, nil) + err = ct.Execute(&bf, data) if err != nil { - return "", fmt.Errorf("Can't read configuration template data: %w", err) + return nil, fmt.Errorf("Can't render template data: %w", err) } - return string(data), nil + return bf.Bytes(), nil } // addAllReplicasToSentinelMonitoring adds all relicas to Sentinel monitoring From 239f89279c343a85cc72a064be7ddd49380bd194 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Fri, 8 Sep 2023 14:16:15 +0300 Subject: [PATCH 07/36] [cli] [core] Add templates validation --- cli/command_validate_templates.go | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 cli/command_validate_templates.go diff --git a/cli/command_validate_templates.go b/cli/command_validate_templates.go new file mode 100644 index 0000000..f961fe8 --- /dev/null +++ b/cli/command_validate_templates.go @@ -0,0 +1,35 @@ +package cli + +// ////////////////////////////////////////////////////////////////////////////////// // +// // +// Copyright (c) 2023 ESSENTIAL KAOS // +// Apache License, Version 2.0 // +// // +// ////////////////////////////////////////////////////////////////////////////////// // + +import ( + "github.com/essentialkaos/ek/v12/fmtc" + "github.com/essentialkaos/ek/v12/terminal" + + CORE "github.com/essentialkaos/rds/core" +) + +// ////////////////////////////////////////////////////////////////////////////////// // + +// ValidateTemplatesCommand is "validate-templates" command handler +func ValidateTemplatesCommand(args CommandArgs) int { + errs := CORE.ValidateTemplates() + + if len(errs) == 0 { + fmtc.Println("{g}Redis and Sentinel configuration templates have no problems{!}") + return EC_OK + } + + terminal.Error("Templates validation errors:\n") + + for _, err := range errs { + terminal.Error("- %v", err) + } + + return EC_ERROR +} From e3fb367af73c7d9a62a26775d1c43d9734c6ac4a Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Fri, 8 Sep 2023 14:27:43 +0300 Subject: [PATCH 08/36] Improve README --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b6a274b..2d9de56 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ RDS supports the next versions of Redis and Sentinel: * `6.0.x` * `6.2.x` -* `7.0.x` **ʀᴇᴄᴏᴍᴍᴇɴᴅᴇᴅ** +* `7.0.x` **← ʀᴇᴄᴏᴍᴍᴇɴᴅᴇᴅ** * `7.2.x` RDS packages do not have Redis as a dependency, so you can install it from any source (_package, sources, prebuilt binaries…_). @@ -45,7 +45,7 @@ RDS packages do not have Redis as a dependency, so you can install it from any s * `redis` * `redis60` * `redis62` -* `redis70` **ʀᴇᴄᴏᴍᴍᴇɴᴅᴇᴅ** +* `redis70` **← ʀᴇᴄᴏᴍᴍᴇɴᴅᴇᴅ** * `redis72`

@@ -68,14 +68,14 @@ Instances commands cli id:db command Run CLI connected to Redis instance cpu id period Calculate instance CPU usage memory id Show instance memory usage - info id section Show system info about Redis instance + info id section… Show system info about Redis instance stats-command id Show statistics based on the command type stats-latency id Show latency statistics based on the command type stats-error id Show error statistics clients id filter Show list of connected clients track id interval Show interactive info about Redis instance - conf id property Show configuration of Redis instance - list filter Show list of all Redis instances + conf id filter… Show configuration of Redis instance + list filter… Show list of all Redis instances stats Show overall statistics top field num Show instances top top-diff file field num Compare current and dumped top data @@ -121,6 +121,7 @@ Common commands help command Show command usage info settings section… Show settings from global configuration file gen-token Generate authentication token for sync daemon + validate-templates Validate Redis and Sentinel templates Options @@ -131,6 +132,7 @@ Options --format, -f format Output format (text/json/xml) --yes, -y Automatically answer yes for all questions --simple, -S Simplify output (useful for copy-paste) + --raw, -R Force raw output (useful for scripts) --no-color, -nc Disable colors in output --help, -h Show this help message --version, -v Show information about version From 5e95322fe21de307503ff2cb595fcab4115615f0 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Sun, 10 Sep 2023 19:27:30 +0300 Subject: [PATCH 09/36] [cli] Add 'backup-*' commands --- README.md | 6 +- cli/cli.go | 30 ++- cli/command_backup.go | 443 ++++++++++++++++++++++++++++++++++++++++++ cli/command_help.go | 64 +++++- 4 files changed, 534 insertions(+), 9 deletions(-) create mode 100644 cli/command_backup.go diff --git a/README.md b/README.md index 2d9de56..29999d0 100644 --- a/README.md +++ b/README.md @@ -82,9 +82,13 @@ Instances commands top-dump file Dump top data to file slowlog-get id num Show last entries from slow log slowlog-reset id Clear slow log - check Check for dead instances + backup-create id Create snapshot of RDB file + backup-restore id Restore instance data from snapshot + backup-clean id Delete all backup snapshots + backup-list id List backup snapshots tag-add id tag Add tag to instance tag-remove id tag Remove tag from instance + check Check for dead instances Superuser commands diff --git a/cli/cli.go b/cli/cli.go index 9cf6a75..6dfe48e 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -83,6 +83,10 @@ const ( // Supported commands const ( + COMMAND_BACKUP_CREATE = "backup-create" + COMMAND_BACKUP_RESTORE = "backup-restore" + COMMAND_BACKUP_CLEAN = "backup-clean" + COMMAND_BACKUP_LIST = "backup-list" COMMAND_BATCH_CREATE = "batch-create" COMMAND_BATCH_EDIT = "batch-edit" COMMAND_CHECK = "check" @@ -447,6 +451,10 @@ func initCommands() { commands[COMMAND_RELOAD] = &CommandRoutine{ReloadCommand, AUTH_SUPERUSER, true, true} commands[COMMAND_REGEN] = &CommandRoutine{RegenCommand, AUTH_SUPERUSER, true, true} commands[COMMAND_MAINTENANCE] = &CommandRoutine{MaintenanceCommand, AUTH_SUPERUSER, true, true} + commands[COMMAND_BACKUP_CREATE] = &CommandRoutine{BackupCreateCommand, AUTH_INSTANCE | AUTH_SUPERUSER, false, true} + commands[COMMAND_BACKUP_RESTORE] = &CommandRoutine{BackupRestoreCommand, AUTH_INSTANCE | AUTH_SUPERUSER, false, true} + commands[COMMAND_BACKUP_CLEAN] = &CommandRoutine{BackupCleanCommand, AUTH_INSTANCE | AUTH_SUPERUSER, false, true} + commands[COMMAND_BACKUP_LIST] = &CommandRoutine{BackupListCommand, AUTH_INSTANCE | AUTH_SUPERUSER, false, true} } if isMaster { @@ -721,10 +729,11 @@ func authenticate(authType AuthType, strict bool, instanceID string) (bool, erro // getSpellcheckModel train spellchecker with supported commands func getSpellcheckModel() *spellcheck.Model { return spellcheck.Train([]string{ - COMMAND_BATCH_CREATE, COMMAND_BATCH_EDIT, COMMAND_CHECK, COMMAND_CLI, - COMMAND_CPU, COMMAND_CONF, COMMAND_CREATE, COMMAND_DELETE, COMMAND_DESTROY, - COMMAND_EDIT, COMMAND_GEN_TOKEN, COMMAND_GO, COMMAND_HELP, COMMAND_INFO, - COMMAND_INIT, COMMAND_KILL, COMMAND_LIST, COMMAND_MAINTENANCE, + COMMAND_BACKUP_CREATE, COMMAND_BACKUP_RESTORE, COMMAND_BACKUP_CLEAN, + COMMAND_BACKUP_LIST, COMMAND_BATCH_CREATE, COMMAND_BATCH_EDIT, COMMAND_CHECK, + COMMAND_CLI, COMMAND_CPU, COMMAND_CONF, COMMAND_CREATE, COMMAND_DELETE, + COMMAND_DESTROY, COMMAND_EDIT, COMMAND_GEN_TOKEN, COMMAND_GO, COMMAND_HELP, + COMMAND_INFO, COMMAND_INIT, COMMAND_KILL, COMMAND_LIST, COMMAND_MAINTENANCE, COMMAND_MEMORY, COMMAND_REGEN, COMMAND_RELEASE, COMMAND_RELOAD, COMMAND_REMOVE, COMMAND_REPLICATION, COMMAND_REPLICATION_ROLE_SET, COMMAND_RESTART, COMMAND_RESTART_ALL, COMMAND_RESTART_ALL_PROP, @@ -880,12 +889,17 @@ func showSmartUsage() { info.AddCommand(COMMAND_TOP_DUMP, "Dump top data to file", "file") info.AddCommand(COMMAND_SLOWLOG_GET, "Show last entries from slow log", "id", "?num") info.AddCommand(COMMAND_SLOWLOG_RESET, "Clear slow log", "id") - info.AddCommand(COMMAND_CHECK, "Check for dead instances") + info.AddCommand(COMMAND_BACKUP_CREATE, "Create snapshot of RDB file", "id") + info.AddCommand(COMMAND_BACKUP_RESTORE, "Restore instance data from snapshot", "id") + info.AddCommand(COMMAND_BACKUP_CLEAN, "Remove all backup snapshots", "id") + info.AddCommand(COMMAND_BACKUP_LIST, "List backup snapshots", "id") if isMaster { info.AddCommand(COMMAND_TAG_ADD, "Add tag to instance", "id", "tag") info.AddCommand(COMMAND_TAG_REMOVE, "Remove tag from instance", "id", "tag") } + + info.AddCommand(COMMAND_CHECK, "Check for dead instances") } info.AddGroup("Superuser commands") @@ -1004,9 +1018,13 @@ func genUsage() *usage.Info { info.AddCommand(COMMAND_TOP_DUMP, "Dump top data to file", "file") info.AddCommand(COMMAND_SLOWLOG_GET, "Show last entries from slow log", "id", "?num") info.AddCommand(COMMAND_SLOWLOG_RESET, "Clear slow log", "id") - info.AddCommand(COMMAND_CHECK, "Check for dead instances") info.AddCommand(COMMAND_TAG_ADD, "Add tag to instance", "id", "tag") info.AddCommand(COMMAND_TAG_REMOVE, "Remove tag from instance", "id", "tag") + info.AddCommand(COMMAND_BACKUP_CREATE, "Create snapshot of RDB file", "id") + info.AddCommand(COMMAND_BACKUP_RESTORE, "Restore instance data from snapshot", "id") + info.AddCommand(COMMAND_BACKUP_CLEAN, "Delete all backup snapshots", "id") + info.AddCommand(COMMAND_BACKUP_LIST, "List backup snapshots", "id") + info.AddCommand(COMMAND_CHECK, "Check for dead instances") info.AddGroup("Superuser commands") info.AddCommand(COMMAND_GO, "Generate superuser access credentials") diff --git a/cli/command_backup.go b/cli/command_backup.go new file mode 100644 index 0000000..1f6108b --- /dev/null +++ b/cli/command_backup.go @@ -0,0 +1,443 @@ +package cli + +// ////////////////////////////////////////////////////////////////////////////////// // +// // +// Copyright (c) 2023 ESSENTIAL KAOS // +// Apache License, Version 2.0 // +// // +// ////////////////////////////////////////////////////////////////////////////////// // + +import ( + "fmt" + "os" + "strconv" + "time" + + "github.com/essentialkaos/ek/v12/fmtc" + "github.com/essentialkaos/ek/v12/fmtutil" + "github.com/essentialkaos/ek/v12/fmtutil/table" + "github.com/essentialkaos/ek/v12/fsutil" + "github.com/essentialkaos/ek/v12/log" + "github.com/essentialkaos/ek/v12/mathutil" + "github.com/essentialkaos/ek/v12/path" + "github.com/essentialkaos/ek/v12/sortutil" + "github.com/essentialkaos/ek/v12/spinner" + "github.com/essentialkaos/ek/v12/strutil" + "github.com/essentialkaos/ek/v12/terminal" + "github.com/essentialkaos/ek/v12/timeutil" + + CORE "github.com/essentialkaos/rds/core" + REDIS "github.com/essentialkaos/rds/redis" +) + +// ////////////////////////////////////////////////////////////////////////////////// // + +const ( + BACKUP_PERMS = 0600 // Default permissions for backups + MAX_BACKUPS = 10 // Maximum number of backups +) + +// ////////////////////////////////////////////////////////////////////////////////// // + +// BackupCreateCommand is "backup-create" command handler +func BackupCreateCommand(args CommandArgs) int { + err := args.Check(false) + + if err != nil { + terminal.Error(err.Error()) + return EC_ERROR + } + + id, _, err := CORE.ParseIDDBPair(args.Get(0)) + + if err != nil { + terminal.Error(err.Error()) + return EC_ERROR + } + + state, err := CORE.GetInstanceState(id, false) + + if err != nil { + terminal.Error("Can't check instance state: %v", err) + return EC_ERROR + } + + backupName := fmt.Sprintf("backup-%d.rdb", time.Now().Unix()) + rdbFile := CORE.GetInstanceRDBPath(id) + backupFile := path.Join(CORE.GetInstanceDataDirPath(id), backupName) + + if state.IsWorks() { + spinner.Show("Saving instance data in background") + + _, err := CORE.ExecCommand(id, &REDIS.Request{ + Command: []string{"BGSAVE"}, + }) + + if err != nil { + spinner.Done(false) + fmtc.NewLine() + terminal.Error("Can't exec BGSAVE command: %v", err) + return EC_ERROR + } + + waitForDump(id) + } else { + if !fsutil.IsExist(rdbFile) { + terminal.Warn("There is no RDB snapshot of instance data") + return EC_ERROR + } + + spinner.Show("Creating snapshot of instance data") + } + + if getBackupNum(id) == MAX_BACKUPS { + err := cleanOldBackups(id, 1) + + if err != nil { + terminal.Error(err.Error()) + return EC_ERROR + } + } + + rdbFileSize := fsutil.GetSize(rdbFile) + + if rdbFileSize > 0 { + spinner.Update( + "Copying snapshot of RDB file {s}(%s){!}", + fmtutil.PrettySize(rdbFileSize), + ) + } else { + spinner.Update("Copying snapshot of RDB file") + } + + err = fsutil.CopyFile(rdbFile, backupFile, BACKUP_PERMS) + + spinner.Done(err == nil) + + if err != nil { + fmtc.NewLine() + terminal.Error("Can't exec BGSAVE command: %v", err) + return EC_ERROR + } + + log.Info("(%s) Created RDB backup of instance with ID %d", CORE.User.RealName, id) + + return EC_OK +} + +// BackupRestoreCommand is "backup-restore" command handler +func BackupRestoreCommand(args CommandArgs) int { + var index int + + err := args.Check(false) + + if err != nil { + terminal.Error(err.Error()) + return EC_ERROR + } + + id, _, err := CORE.ParseIDDBPair(args.Get(0)) + + if err != nil { + terminal.Error(err.Error()) + return EC_ERROR + } + + state, err := CORE.GetInstanceState(id, false) + + if err != nil { + terminal.Error("Can't check instance state: %v", err) + return EC_ERROR + } + + if state.IsWorks() { + terminal.Warn("Instance must be stopped for restoring data from snapshot") + return EC_WARN + } + + numBackups := getBackupNum(id) + + if numBackups == 0 { + terminal.Warn("There are no snapshots of given instance data") + return EC_WARN + } + + listBackups(id) + + fmtc.NewLine() + + for { + selectedIndex, err := terminal.Read(fmt.Sprintf("Enter index of snapshot to restore (1-%d)", numBackups), true) + + if err != nil { + return EC_OK + } + + fmtc.NewLine() + + index, err = strconv.Atoi(selectedIndex) + + if err != nil { + terminal.Error("Invalid snapshot index: %v\n", err) + continue + } + + if index > numBackups || index < 1 { + terminal.Error("There is no backup with index %d\n", index) + continue + } + + break + } + + snapshots := getBackupFiles(id) + + if len(snapshots) == 0 { + terminal.Error("Can't get list of snapshots") + return EC_ERROR + } + + spinner.Show("Restoring instance data from snapshot") + + err = restoreBackupSnapshot(id, snapshots[index-1]) + + spinner.Done(err == nil) + + if err != nil { + fmtc.NewLine() + terminal.Error("Can't restore snapshot: %v", err) + log.Error("(%s) Tried to restore snapshot of instance with ID %d, but got error: %v", CORE.User.RealName, id, err) + return EC_ERROR + } + + log.Info("(%s) Restored RDB backup of instance with ID %d", CORE.User.RealName, id) + + return EC_OK +} + +// BackupCleanCommand is "backup-clean" command handler +func BackupCleanCommand(args CommandArgs) int { + err := args.Check(false) + + if err != nil { + terminal.Error(err.Error()) + return EC_ERROR + } + + id, _, err := CORE.ParseIDDBPair(args.Get(0)) + + if err != nil { + terminal.Error(err.Error()) + return EC_ERROR + } + + numBackups := getBackupNum(id) + + if numBackups == 0 { + terminal.Warn("There are no snapshots of given instance data") + return EC_WARN + } + + listBackups(id) + + fmtc.NewLine() + + ok, _ := terminal.ReadAnswer("Delete all these snapshots?", "N") + + if !ok { + return EC_OK + } + + fmtc.NewLine() + + spinner.Show("Removing instance backups {s}(%d){!}", numBackups) + + err = cleanOldBackups(id, MAX_BACKUPS) + + spinner.Done(err == nil) + + if err != nil { + fmtc.NewLine() + terminal.Error("Can't clean backups: %v", err) + log.Error("(%s) Tried to remove RDB backups (%d) of instance with ID %d, but got error: %v", CORE.User.RealName, numBackups, id, err) + return EC_ERROR + } + + log.Info("(%s) Removed RDB backups (%d) of instance with ID %d", CORE.User.RealName, numBackups, id) + + return EC_OK +} + +// BackupListCommand is "backup-list" command handler +func BackupListCommand(args CommandArgs) int { + err := args.Check(false) + + if err != nil { + terminal.Error(err.Error()) + return EC_ERROR + } + + id, _, err := CORE.ParseIDDBPair(args.Get(0)) + + if err != nil { + terminal.Error(err.Error()) + return EC_ERROR + } + + if getBackupNum(id) == 0 { + terminal.Warn("There are no snapshots of given instance data") + return EC_WARN + } + + listBackups(id) + + return EC_OK +} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// getBackupNum returns number of backups for instance with given ID +func getBackupNum(id int) int { + return len(getBackupFiles(id)) +} + +// getBackupFiles returns slice with paths to backup files +func getBackupFiles(id int) []string { + files := fsutil.List( + CORE.GetInstanceDataDirPath(id), false, + fsutil.ListingFilter{MatchPatterns: []string{"backup-*.rdb"}}, + ) + + sortutil.StringsNatural(files) + + return files +} + +// listBackups shows table with information about all backups of given instance +func listBackups(id int) { + if getBackupNum(id) == 0 { + return + } + + dataDir := CORE.GetInstanceDataDirPath(id) + t := table.NewTable().SetHeaders("#", "SIZE", "DATE") + + for index, file := range getBackupFiles(id) { + t.Add( + fmt.Sprintf("{s}%d{!}", index+1), + fmtutil.PrettySize(fsutil.GetSize(path.Join(dataDir, file))), + extractBackupDateFromFilename(file), + ) + } + + t.Render() +} + +// cleanOldBackups remove old backups for instance with given ID +func cleanOldBackups(id, numOldBackups int) error { + numOldBackups = mathutil.Between(numOldBackups, 1, MAX_BACKUPS) + + dataDir := CORE.GetInstanceDataDirPath(id) + + for index, file := range getBackupFiles(id) { + err := os.Remove(path.Join(dataDir, file)) + + if err != nil { + return fmt.Errorf("Can't remove backup file: %w", err) + } + + if index+1 == numOldBackups { + break + } + } + + return nil +} + +// restoreBackupSnapshot replaces default dump file with snapshot +func restoreBackupSnapshot(id int, snapshot string) error { + rdbFile := CORE.GetInstanceRDBPath(id) + dataDir := CORE.GetInstanceDataDirPath(id) + snapshotFile := path.Join(dataDir, snapshot) + + uid, gid, err := fsutil.GetOwner(dataDir) + + if err != nil { + return fmt.Errorf("Can't get data directory owner: %v", err) + } + + err = fsutil.CopyFile(snapshotFile, rdbFile, 0640) + + if err != nil { + return fmt.Errorf("Can't copy snapshot: %v", err) + } + + err = os.Chown(rdbFile, uid, gid) + + if err != nil { + return fmt.Errorf("Can't change RDB file owner: %v", err) + } + + return nil +} + +// waitForDump waits until RDB file updated +func waitForDump(id int) { + var tempRDB string + + rdbFile := CORE.GetInstanceRDBPath(id) + modTime, _ := fsutil.GetMTime(rdbFile) + + for range time.NewTicker(time.Second).C { + curModTime, _ := fsutil.GetMTime(rdbFile) + + if !curModTime.IsZero() && !curModTime.Equal(modTime) { + return + } + + if tempRDB == "" { + tempRDB = getTemporaryRDBPath(id) + } + + if tempRDB != "" { + tempRDBSize := fsutil.GetSize(tempRDB) + + if tempRDBSize > 0 { + spinner.Update( + "Saving instance data in background {s}(%s){!}", + fmtutil.PrettySize(tempRDBSize), + ) + } + } + } +} + +// extractBackupDateFromFilename extracts backup creation date from filename +func extractBackupDateFromFilename(file string) string { + ts := strutil.Exclude(file, "backup-") + ts = strutil.Exclude(ts, ".rdb") + tsi, _ := strconv.ParseInt(ts, 10, 64) + + if tsi == 0 { + return "unknown" + } + + return timeutil.Format(time.Unix(tsi, 0), "%Y/%m/%d %H:%M:%S") +} + +// getTemporaryRDB returns path to temporary RDB file +func getTemporaryRDBPath(id int) string { + files := fsutil.List( + CORE.GetInstanceDataDirPath(id), false, + fsutil.ListingFilter{ + MatchPatterns: []string{"temp-*.rdb"}, + MTimeYounger: time.Now().Unix() - 60, + }, + ) + + if len(files) == 0 { + return "" + } + + return path.Join(CORE.GetInstanceDataDirPath(id), files[0]) +} diff --git a/cli/command_help.go b/cli/command_help.go index bde07ea..25a817d 100644 --- a/cli/command_help.go +++ b/cli/command_help.go @@ -47,6 +47,10 @@ type helpInfo struct { func HelpCommand(args CommandArgs) int { commandName := args.Get(0) commandList := map[string]func(){ + COMMAND_BACKUP_CREATE: helpCommandBackupCreate, + COMMAND_BACKUP_RESTORE: helpCommandBackupRestore, + COMMAND_BACKUP_CLEAN: helpCommandBackupClean, + COMMAND_BACKUP_LIST: helpCommandBackupList, COMMAND_BATCH_CREATE: helpCommandBatchCreate, COMMAND_BATCH_EDIT: helpCommandBatchEdit, COMMAND_CHECK: helpCommandCheck, @@ -930,6 +934,62 @@ func helpCommandValidateTemplates() { }.render() } +// helpCommandBackupCreate prints info about "backup-create" command usage +func helpCommandBackupCreate() { + helpInfo{ + command: COMMAND_BACKUP_CREATE, + desc: "Create snapshot of RDB file.", + arguments: []helpInfoArgument{ + {"id", "Instance unique ID", false}, + }, + examples: []helpInfoExample{ + {"", "7", "Create an RDB file snapshot of the instance with the ID 7"}, + }, + }.render() +} + +// helpCommandBackupRestore prints info about "backup-restore" command usage +func helpCommandBackupRestore() { + helpInfo{ + command: COMMAND_BACKUP_RESTORE, + desc: "Restore previously created snapshot of instance data.", + arguments: []helpInfoArgument{ + {"id", "Instance unique ID", false}, + }, + examples: []helpInfoExample{ + {"", "7", "Restore a data snapshot of the instance with the ID 7"}, + }, + }.render() +} + +// helpCommandBackupClean prints info about "backup-clean" command usage +func helpCommandBackupClean() { + helpInfo{ + command: COMMAND_BACKUP_CLEAN, + desc: "Delete all backup snapshots.", + arguments: []helpInfoArgument{ + {"id", "Instance unique ID", false}, + }, + examples: []helpInfoExample{ + {"", "7", "Delete all RDB snapshots of the instance with the ID 7"}, + }, + }.render() +} + +// helpCommandBackupList prints info about "backup-list" command usage +func helpCommandBackupList() { + helpInfo{ + command: COMMAND_BACKUP_LIST, + desc: "Show information regarding all backup snapshots.", + arguments: []helpInfoArgument{ + {"id", "Instance unique ID", false}, + }, + examples: []helpInfoExample{ + {"", "7", "Show information about all snapshots of the instance with the ID 7"}, + }, + }.render() +} + // ////////////////////////////////////////////////////////////////////////////////// // // getNiceOptions parse option and return formatted string @@ -984,7 +1044,7 @@ func (i helpInfo) renderUsage() { arguments = append(arguments, cmdArg.name) } - fmtc.Printf(" rds {y}%s{!} {c}%s{!}\n\n", i.command, strings.Join(arguments, " ")) + fmtc.Printf(" rds {y}%s{!} {#45}%s{!}\n\n", i.command, strings.Join(arguments, " ")) } // renderDescription render description @@ -1006,7 +1066,7 @@ func (i helpInfo) renderArguments() { fmtStr := getArgumentFormatting(i.arguments) for _, argument := range i.arguments { - fmtc.Printf(" {c}"+fmtStr+"{!} %s", argument.name, argument.desc) + fmtc.Printf(" {#45}"+fmtStr+"{!} %s", argument.name, argument.desc) if argument.optional { fmtc.Printf(" {s-}(optional){!}") From 7045ec1c41f7247709544b796a19a188cadb94a9 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Sun, 10 Sep 2023 19:29:21 +0300 Subject: [PATCH 10/36] [core] Run all processes with umask 027 --- core/core.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/core.go b/core/core.go index 4cf379b..b237aa8 100644 --- a/core/core.go +++ b/core/core.go @@ -3169,6 +3169,9 @@ func runAsUser(user, logFile string, args ...string) error { defer logFd.Close() + oldUMask := syscall.Umask(027) + defer syscall.Umask(oldUMask) + w := bufio.NewWriter(logFd) cmdArgs := []string{"-s", "/bin/bash", user, "-c", strings.Join(args, " ")} cmd := exec.Command(BIN_RUNUSER, cmdArgs...) From c40c0390ce6f7b0220f442c1dc1ffeabb042a858 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Sun, 10 Sep 2023 22:35:16 +0300 Subject: [PATCH 11/36] [cli] Actions logging improved --- cli/cli.go | 5 +++++ cli/command_backup.go | 13 ++++++------ cli/command_batch_create.go | 2 ++ cli/command_batch_edit.go | 3 +-- cli/command_create.go | 8 ++------ cli/command_destroy.go | 3 +-- cli/command_edit.go | 3 +-- cli/command_kill.go | 3 +-- cli/command_maintenance.go | 2 ++ cli/command_regen.go | 9 +++++---- cli/command_reload.go | 11 +++++----- cli/command_replication_role_set.go | 5 ++--- cli/command_restart.go | 3 +-- cli/command_restart_all.go | 4 ++-- cli/command_sentinel.go | 9 ++++----- cli/command_slowlog.go | 1 + cli/command_start.go | 4 ++-- cli/command_start_all.go | 8 +++----- cli/command_state.go | 31 +++++++++++++---------------- cli/command_stop.go | 3 +-- cli/command_stop_all.go | 8 +++----- cli/command_tag.go | 6 +++--- 22 files changed, 68 insertions(+), 76 deletions(-) diff --git a/cli/cli.go b/cli/cli.go index 6dfe48e..22d5cdd 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -218,6 +218,9 @@ var safeCommands = []string{ COMMAND_SETTINGS, } +// logger is CLI logger +var logger *Logger + // commands is list of command handlers var commands map[string]*CommandRoutine @@ -389,6 +392,8 @@ func setupLogger() { terminal.Error(err.Error()) os.Exit(EC_ERROR) } + + logger = &Logger{} } // disableProxy disable proxy for requests to sync daemon diff --git a/cli/command_backup.go b/cli/command_backup.go index 1f6108b..b7b797d 100644 --- a/cli/command_backup.go +++ b/cli/command_backup.go @@ -17,7 +17,6 @@ import ( "github.com/essentialkaos/ek/v12/fmtutil" "github.com/essentialkaos/ek/v12/fmtutil/table" "github.com/essentialkaos/ek/v12/fsutil" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/mathutil" "github.com/essentialkaos/ek/v12/path" "github.com/essentialkaos/ek/v12/sortutil" @@ -116,11 +115,11 @@ func BackupCreateCommand(args CommandArgs) int { if err != nil { fmtc.NewLine() - terminal.Error("Can't exec BGSAVE command: %v", err) + terminal.Error("Can't copy snapshot file: %v", err) return EC_ERROR } - log.Info("(%s) Created RDB backup of instance with ID %d", CORE.User.RealName, id) + logger.Info(id, "Created RDB backup") return EC_OK } @@ -206,11 +205,11 @@ func BackupRestoreCommand(args CommandArgs) int { if err != nil { fmtc.NewLine() terminal.Error("Can't restore snapshot: %v", err) - log.Error("(%s) Tried to restore snapshot of instance with ID %d, but got error: %v", CORE.User.RealName, id, err) + logger.Error(id, "Tried to restore snapshot, but got error: %v", err) return EC_ERROR } - log.Info("(%s) Restored RDB backup of instance with ID %d", CORE.User.RealName, id) + logger.Info(id, "Restored RDB backup") return EC_OK } @@ -259,11 +258,11 @@ func BackupCleanCommand(args CommandArgs) int { if err != nil { fmtc.NewLine() terminal.Error("Can't clean backups: %v", err) - log.Error("(%s) Tried to remove RDB backups (%d) of instance with ID %d, but got error: %v", CORE.User.RealName, numBackups, id, err) + logger.Error(id, "Tried to remove RDB backups (%d), but got error: %v", numBackups, err) return EC_ERROR } - log.Info("(%s) Removed RDB backups (%d) of instance with ID %d", CORE.User.RealName, numBackups, id) + logger.Info(id, "Removed RDB backups (%d)", numBackups) return EC_OK } diff --git a/cli/command_batch_create.go b/cli/command_batch_create.go index 93a8bbb..60a127f 100644 --- a/cli/command_batch_create.go +++ b/cli/command_batch_create.go @@ -98,6 +98,8 @@ func BatchCreateCommand(args CommandArgs) int { continue } + logger.Info(meta.ID, "Instance created (batch)") + err = SC.PropagateCommand(API.COMMAND_CREATE, meta.ID, meta.UUID) if err != nil { diff --git a/cli/command_batch_edit.go b/cli/command_batch_edit.go index d4e57a4..2078d33 100644 --- a/cli/command_batch_edit.go +++ b/cli/command_batch_edit.go @@ -14,7 +14,6 @@ import ( "strings" "github.com/essentialkaos/ek/v12/fmtc" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/pluralize" "github.com/essentialkaos/ek/v12/spinner" "github.com/essentialkaos/ek/v12/strutil" @@ -117,7 +116,7 @@ func BatchEditCommand(args CommandArgs) int { continue } - log.Info("(%s) Updated info for instance with ID %d", CORE.User.RealName, id) + logger.Info(id, "Instance info updated (batch)") err = SC.PropagateCommand(API.COMMAND_EDIT, meta.ID, meta.UUID) diff --git a/cli/command_create.go b/cli/command_create.go index 5e3a6f8..e0a11a4 100644 --- a/cli/command_create.go +++ b/cli/command_create.go @@ -14,7 +14,6 @@ import ( "github.com/essentialkaos/ek/v12/fmtc" "github.com/essentialkaos/ek/v12/fmtutil" "github.com/essentialkaos/ek/v12/fmtutil/table" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/options" "github.com/essentialkaos/ek/v12/strutil" "github.com/essentialkaos/ek/v12/system" @@ -108,12 +107,9 @@ func CreateCommand(args CommandArgs) int { fmtc.Println("{*}Done, a new Redis instance has been successfully created. Just for you.{!}") if len(tags) == 0 { - log.Info("(%s) Created instance with ID %d", CORE.User.RealName, meta.ID) + logger.Info(meta.ID, "Instance created") } else { - log.Info( - "(%s) Created instance with ID %d (tags: %s)", - CORE.User.RealName, meta.ID, strings.Join(tags, ","), - ) + logger.Info(meta.ID, "Instance created (tags: %s)", strings.Join(tags, ",")) } showInstanceInfo(meta, info, tags) diff --git a/cli/command_destroy.go b/cli/command_destroy.go index a4632b5..c42e00a 100644 --- a/cli/command_destroy.go +++ b/cli/command_destroy.go @@ -9,7 +9,6 @@ package cli import ( "github.com/essentialkaos/ek/v12/fmtc" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/terminal" API "github.com/essentialkaos/rds/api" @@ -73,7 +72,7 @@ func DestroyCommand(args CommandArgs) int { return EC_ERROR } - log.Info("(%s) Destroyed instance with ID %d", CORE.User.RealName, id) + logger.Info(id, "Instance destroyed") fmtc.Printf("{*}Done. Instance with ID %d successfully destroyed.{!}\n", id) err = SC.PropagateCommand(API.COMMAND_DESTROY, id, meta.UUID) diff --git a/cli/command_edit.go b/cli/command_edit.go index 5d59c17..1bbd1bb 100644 --- a/cli/command_edit.go +++ b/cli/command_edit.go @@ -9,7 +9,6 @@ package cli import ( "github.com/essentialkaos/ek/v12/fmtc" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/sliceutil" "github.com/essentialkaos/ek/v12/system" "github.com/essentialkaos/ek/v12/terminal" @@ -112,7 +111,7 @@ func EditCommand(args CommandArgs) int { return EC_ERROR } - log.Info("(%s) Updated info for instance with ID %d", CORE.User.RealName, id) + logger.Info(id, "Instance info updated") fmtc.Printf("{g}Done. Data for instance with ID %d successfully updated.{!}\n", id) err = SC.PropagateCommand(API.COMMAND_EDIT, meta.ID, meta.UUID) diff --git a/cli/command_kill.go b/cli/command_kill.go index 16de5d3..517a745 100644 --- a/cli/command_kill.go +++ b/cli/command_kill.go @@ -9,7 +9,6 @@ package cli import ( "github.com/essentialkaos/ek/v12/fmtc" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/spinner" "github.com/essentialkaos/ek/v12/terminal" @@ -76,7 +75,7 @@ func KillCommand(args CommandArgs) int { return EC_ERROR } - log.Info("(%s) Killed instance with ID %d", CORE.User.RealName, id) + logger.Info(id, "Instance killed") err = CORE.SaveStates(CORE.GetStatesFilePath()) diff --git a/cli/command_maintenance.go b/cli/command_maintenance.go index a79dc4b..1f05a8f 100644 --- a/cli/command_maintenance.go +++ b/cli/command_maintenance.go @@ -30,12 +30,14 @@ func MaintenanceCommand(args CommandArgs) int { err = createMaintenanceLock() if err == nil { fmtc.Println("Maintenance mode is {g}enabled{!}") + logger.Info(-1, "Maintenance mode enabled") } case "disable", "no": err = removeMaintenanceLock() if err == nil { fmtc.Println("Maintenance mode is {y}disabled{!}") + logger.Info(-1, "Maintenance mode disabled") } default: diff --git a/cli/command_regen.go b/cli/command_regen.go index a8b2636..831f9d5 100644 --- a/cli/command_regen.go +++ b/cli/command_regen.go @@ -9,7 +9,6 @@ package cli import ( "github.com/essentialkaos/ek/v12/fmtc" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/spinner" "github.com/essentialkaos/ek/v12/terminal" @@ -94,10 +93,11 @@ func regenerateInstanceConfig(id int) int { if err != nil { fmtc.NewLine() terminal.Error(err.Error()) + logger.Error(id, "Configuration file regeneration error: %v", err) return EC_ERROR } - log.Info("(%s) Regenerated configuration file for instance %d", CORE.User.RealName, id) + logger.Info(id, "Configuration file regenerated") return EC_OK } @@ -128,11 +128,12 @@ func regenerateAllConfigs() int { if err != nil { fmtc.NewLine() terminal.Error(err.Error()) + logger.Error(id, "Configuration file regeneration (batch) error: %v", err) return EC_ERROR + } else { + logger.Info(id, "Configuration file regenerated (batch)") } } - log.Info("(%s) Regenerated configuration files for all instances", CORE.User.RealName) - return EC_OK } diff --git a/cli/command_reload.go b/cli/command_reload.go index d3d3e24..b50bd67 100644 --- a/cli/command_reload.go +++ b/cli/command_reload.go @@ -13,7 +13,6 @@ import ( "github.com/essentialkaos/ek/v12/fmtc" "github.com/essentialkaos/ek/v12/fmtutil/panel" "github.com/essentialkaos/ek/v12/fmtutil/table" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/spinner" "github.com/essentialkaos/ek/v12/terminal" @@ -136,10 +135,9 @@ Use {*}REPLICAOF{!} or {*}SLAVEOF{!} commands for changing replication settings. errs := CORE.ReloadInstanceConfig(id) - log.Info("(%s) Reloaded configuration for instance %d", CORE.User.RealName, id) - if len(errs) == 0 { fmtc.Println("{g}Instance configuration successfully updated{!}") + logger.Info(id, "Instance configuration reloaded") return EC_OK } @@ -147,6 +145,8 @@ Use {*}REPLICAOF{!} or {*}SLAVEOF{!} commands for changing replication settings. terminal.Error(err.Error()) } + logger.Error(id, "Instance configuration reloaded with errors (%d)", len(errs)) + return EC_ERROR } @@ -187,11 +187,12 @@ func reloadAllConfigs() int { } fmtc.NewLine() hasErrors = true + logger.Error(id, "Instance configuration reloaded (batch) with errors (%d)", len(errs)) + } else { + logger.Info(id, "Instance configuration reloaded (batch)") } } - log.Info("(%s) Reloaded configuration for all instances", CORE.User.RealName) - if hasErrors { return EC_ERROR } diff --git a/cli/command_replication_role_set.go b/cli/command_replication_role_set.go index 6851fec..2c05774 100644 --- a/cli/command_replication_role_set.go +++ b/cli/command_replication_role_set.go @@ -11,7 +11,6 @@ import ( "time" "github.com/essentialkaos/ek/v12/fmtc" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/spinner" "github.com/essentialkaos/ek/v12/terminal" @@ -98,7 +97,7 @@ func setRoleFromMasterToMinion() int { return EC_ERROR } - log.Info("(%s) Reconfigurated node to minion role", CORE.User.RealName) + logger.Info(-1, "Node reconfigurated to minion role") return EC_OK } @@ -115,7 +114,7 @@ func setRoleFromMinionToMaster() int { return EC_ERROR } - log.Info("(%s) Reconfigurated node to master role", CORE.User.RealName) + logger.Info(-1, "Node reconfigurated to minion role") return EC_OK } diff --git a/cli/command_restart.go b/cli/command_restart.go index 390ad26..80c3f5c 100644 --- a/cli/command_restart.go +++ b/cli/command_restart.go @@ -9,7 +9,6 @@ package cli import ( "github.com/essentialkaos/ek/v12/fmtc" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/spinner" "github.com/essentialkaos/ek/v12/terminal" @@ -96,7 +95,7 @@ func RestartCommand(args CommandArgs) int { return EC_ERROR } - log.Info("(%s) Restarted instance with ID %d (force: %t)", CORE.User.RealName, id, force) + logger.Info(id, "Instance restarted (force: %t)", force) return EC_OK } diff --git a/cli/command_restart_all.go b/cli/command_restart_all.go index bc295c2..06165e8 100644 --- a/cli/command_restart_all.go +++ b/cli/command_restart_all.go @@ -120,11 +120,11 @@ func RestartAllCommand(args CommandArgs) int { continue } + logger.Info(id, "Instance restarted (batch)") + spinner.Done(true) } - log.Info("(%s) Restarted all working instances", CORE.User.RealName) - err = CORE.SaveStates(CORE.GetStatesFilePath()) if err != nil { diff --git a/cli/command_sentinel.go b/cli/command_sentinel.go index b16cb72..da23677 100644 --- a/cli/command_sentinel.go +++ b/cli/command_sentinel.go @@ -10,7 +10,6 @@ package cli import ( "github.com/essentialkaos/ek/v12/fmtc" "github.com/essentialkaos/ek/v12/fmtutil/table" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/spinner" "github.com/essentialkaos/ek/v12/terminal" @@ -52,7 +51,7 @@ func SentinelStartCommand(args CommandArgs) int { return EC_ERROR } - log.Info("(%s) Started Sentinel", CORE.User.RealName) + logger.Info(-1, "Started Sentinel") return EC_OK } @@ -81,7 +80,7 @@ func SentinelStopCommand(args CommandArgs) int { return EC_ERROR } - log.Info("(%s) Stopped Sentinel", CORE.User.RealName) + logger.Info(-1, "Stopped Sentinel") return EC_OK } @@ -102,7 +101,7 @@ func SentinelResetCommand(args CommandArgs) int { return EC_ERROR } - log.Info("(%s) Reset Sentinel state for all instances", CORE.User.RealName) + logger.Info(-1, "Sentinel state has been reset for all instances") return EC_OK } @@ -143,7 +142,7 @@ func SentinelSwitchMasterCommand(args CommandArgs) int { return EC_ERROR } - log.Info("(%s) Switched master role in Sentinel to current node", CORE.User.RealName) + logger.Info(-1, "Switched master role in Sentinel to current node") return EC_OK } diff --git a/cli/command_slowlog.go b/cli/command_slowlog.go index 5eb4d0f..0c2a935 100644 --- a/cli/command_slowlog.go +++ b/cli/command_slowlog.go @@ -148,6 +148,7 @@ func slowlogReset(id int) int { return EC_ERROR } + logger.Info(id, "Slow log reseted") fmtc.Printf("{g}Slow log successfully cleared for instance %d{!}\n", id) return EC_OK diff --git a/cli/command_start.go b/cli/command_start.go index 41bd455..fea9c63 100644 --- a/cli/command_start.go +++ b/cli/command_start.go @@ -9,7 +9,6 @@ package cli import ( "github.com/essentialkaos/ek/v12/fmtc" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/spinner" "github.com/essentialkaos/ek/v12/terminal" @@ -66,10 +65,11 @@ func StartCommand(args CommandArgs) int { if err != nil { fmtc.NewLine() terminal.Error(err.Error()) + logger.Error(id, "Instance starting error: %v", err) return EC_ERROR } - log.Info("(%s) Started instance with ID %d", CORE.User.RealName, id) + logger.Info(id, "Instance started") err = CORE.SaveStates(CORE.GetStatesFilePath()) diff --git a/cli/command_start_all.go b/cli/command_start_all.go index 6173edb..b968019 100644 --- a/cli/command_start_all.go +++ b/cli/command_start_all.go @@ -11,7 +11,6 @@ import ( "time" "github.com/essentialkaos/ek/v12/fmtc" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/spinner" "github.com/essentialkaos/ek/v12/terminal" @@ -42,8 +41,6 @@ func StartAllCommand(args CommandArgs) int { return EC_WARN } - log.Info("(%s) Initiated the starting of all stopped and dead instances", CORE.User.RealName) - hasErrors := false for _, id := range idList { @@ -69,15 +66,16 @@ func StartAllCommand(args CommandArgs) int { if err != nil { spinner.Done(false) hasErrors = true + logger.Info(id, "Instance starting (batch) error: %v", err) continue } } + logger.Info(id, "Instance started (batch)") + spinner.Done(true) } - log.Info("(%s) Started all stopped instances", CORE.User.RealName) - err = CORE.SaveStates(CORE.GetStatesFilePath()) if err != nil { diff --git a/cli/command_state.go b/cli/command_state.go index ade8878..ea7ccd0 100644 --- a/cli/command_state.go +++ b/cli/command_state.go @@ -13,7 +13,6 @@ import ( "github.com/essentialkaos/ek/v12/fmtc" "github.com/essentialkaos/ek/v12/fsutil" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/spinner" "github.com/essentialkaos/ek/v12/terminal" @@ -53,6 +52,8 @@ func SaveStateCommand(args CommandArgs) int { spinner.Done(true) + logger.Info(-1, "States saved to file %s", statesFile) + return EC_OK } @@ -96,21 +97,22 @@ func RestoreStateCommand(args CommandArgs) int { return EC_ERROR } - log.Info("(%s) Started states restoring from file %s", CORE.User.RealName, path.Clean(statesFile)) + logger.Info(-1, "Started states restoring from file %s", path.Clean(statesFile)) - restored, hasErrors := false, false + restored := false for _, stateInfo := range statesInfo.States { - if !CORE.IsInstanceExist(stateInfo.ID) { + id := stateInfo.ID + + if !CORE.IsInstanceExist(id) { continue } - spinner.Show("Restoring instance %d state", stateInfo.ID) - state, err := CORE.GetInstanceState(stateInfo.ID, false) + spinner.Show("Restoring instance %d state", id) + state, err := CORE.GetInstanceState(id, false) if err != nil { spinner.Done(false) - hasErrors = true continue } @@ -121,31 +123,26 @@ func RestoreStateCommand(args CommandArgs) int { switch { case stateInfo.State.IsWorks(): - err = CORE.StartInstance(stateInfo.ID, true) + err = CORE.StartInstance(id, true) case stateInfo.State.IsStopped(): - err = CORE.StopInstance(stateInfo.ID, false) + err = CORE.StopInstance(id, false) } spinner.Done(err == nil) if err == nil { restored = true + logger.Info(id, "Instance state restored") } else { - hasErrors = true + logger.Info(id, "Instance state restored with error: %v", err) } } if !restored { - log.Info("(%s) No actions are made while states restoring", CORE.User.RealName) + logger.Info(-1, "No actions were made while states restoring") return EC_OK } - if hasErrors { - log.Info("(%s) Restored states from file %s with errors", CORE.User.RealName, path.Clean(statesFile)) - } else { - log.Info("(%s) Restored states from file %s", CORE.User.RealName, path.Clean(statesFile)) - } - err = CORE.SaveStates(CORE.GetStatesFilePath()) if err != nil { diff --git a/cli/command_stop.go b/cli/command_stop.go index 0c2287e..491524d 100644 --- a/cli/command_stop.go +++ b/cli/command_stop.go @@ -9,7 +9,6 @@ package cli import ( "github.com/essentialkaos/ek/v12/fmtc" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/spinner" "github.com/essentialkaos/ek/v12/terminal" @@ -81,7 +80,7 @@ func StopCommand(args CommandArgs) int { return EC_ERROR } - log.Info("(%s) Stopped instance with ID %d (force: %t)", CORE.User.RealName, id, force) + logger.Info(id, "Instance stopped (force: %t)", force) err = CORE.SaveStates(CORE.GetStatesFilePath()) diff --git a/cli/command_stop_all.go b/cli/command_stop_all.go index 5ce42bd..9e99144 100644 --- a/cli/command_stop_all.go +++ b/cli/command_stop_all.go @@ -9,7 +9,6 @@ package cli import ( "github.com/essentialkaos/ek/v12/fmtc" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/spinner" "github.com/essentialkaos/ek/v12/terminal" @@ -109,8 +108,6 @@ func stopAllInstances(idList []int) int { return EC_OK } - log.Info("(%s) Initiated the stopping of all working instances", CORE.User.RealName) - for _, id := range idList { meta, err := CORE.GetInstanceMeta(id) @@ -125,14 +122,15 @@ func stopAllInstances(idList []int) int { if err != nil { spinner.Done(false) hasErrors = true + logger.Info(id, "Instance stopping (batch) error: %v", err) continue } + logger.Info(id, "Instance stopped (batch)") + spinner.Done(true) } - log.Info("(%s) Stopped all working instances", CORE.User.RealName) - err := CORE.SaveStates(CORE.GetStatesFilePath()) if err != nil { diff --git a/cli/command_tag.go b/cli/command_tag.go index 5114b2b..cfa0856 100644 --- a/cli/command_tag.go +++ b/cli/command_tag.go @@ -9,7 +9,6 @@ package cli import ( "github.com/essentialkaos/ek/v12/fmtc" - "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/terminal" API "github.com/essentialkaos/rds/api" @@ -64,7 +63,8 @@ func TagAddCommand(args CommandArgs) int { // We use renderTags function from listing fmtc.Printf("Tag "+renderTags(tag)+" added to instance %d\n", id) - log.Info("(%s) Added tag \"%s\" to instance %d", CORE.User.RealName, tag, id) + + logger.Info(id, "Tag %q added", tag) return EC_OK } @@ -116,7 +116,7 @@ func TagRemoveCommand(args CommandArgs) int { fmtc.Printf("{g}Tag \"%s\" removed from instance %d{!}\n", tagName, id) - log.Info("(%s) Removed tag \"%s\" from instance %d", CORE.User.RealName, tagName, id) + logger.Info(id, "Tag %q removed", tagName) return EC_OK } From 20b4fcae8ab87215a4acaa92a3a9db4581e15279 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Sun, 10 Sep 2023 22:35:27 +0300 Subject: [PATCH 12/36] [cli] Actions logging improved --- cli/logger.go | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 cli/logger.go diff --git a/cli/logger.go b/cli/logger.go new file mode 100644 index 0000000..08875a2 --- /dev/null +++ b/cli/logger.go @@ -0,0 +1,53 @@ +package cli + +// ////////////////////////////////////////////////////////////////////////////////// // +// // +// Copyright (c) 2023 ESSENTIAL KAOS // +// Apache License, Version 2.0 // +// // +// ////////////////////////////////////////////////////////////////////////////////// // + +import ( + "fmt" + + "github.com/essentialkaos/ek/v12/log" + + CORE "github.com/essentialkaos/rds/core" +) + +// ////////////////////////////////////////////////////////////////////////////////// // + +type Logger struct{} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// Error writes debug message to CLI log +func (l *Logger) Debug(id int, f string, a ...any) error { + return log.Debug(l.getPrefix(id)+f, a...) +} + +// Error writes info message to CLI log +func (l *Logger) Info(id int, f string, a ...any) error { + return log.Info(l.getPrefix(id)+f, a...) +} + +// Error writes warning message to CLI log +func (l *Logger) Warn(id int, f string, a ...any) error { + return log.Warn(l.getPrefix(id)+f, a...) +} + +// Error writes error message to CLI log +func (l *Logger) Error(id int, f string, a ...any) error { + return log.Error(l.getPrefix(id)+f, a...) +} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// getPrefix returns log message prefix +func (l *Logger) getPrefix(id int) string { + if id <= 0 { + return fmt.Sprintf("(—|%s) ", CORE.User.RealName) + } + + return fmt.Sprintf("(%d|%s) ", id, CORE.User.RealName) +} From 3addb0dde8dd30c56cd472400fade4172e90f5d0 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Sun, 10 Sep 2023 22:36:08 +0300 Subject: [PATCH 13/36] Fix typos --- cli/command_slowlog.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/command_slowlog.go b/cli/command_slowlog.go index 0c2a935..cd8e884 100644 --- a/cli/command_slowlog.go +++ b/cli/command_slowlog.go @@ -148,7 +148,7 @@ func slowlogReset(id int) int { return EC_ERROR } - logger.Info(id, "Slow log reseted") + logger.Info(id, "Slow log reset") fmtc.Printf("{g}Slow log successfully cleared for instance %d{!}\n", id) return EC_OK From 38ed2d855a1b88d574716b4e6c1100249ff9fbf7 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Sun, 10 Sep 2023 22:54:47 +0300 Subject: [PATCH 14/36] [cli] Improve password auth --- cli/cli.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cli/cli.go b/cli/cli.go index 22d5cdd..db01fc8 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -710,9 +710,13 @@ func authenticate(authType AuthType, strict bool, instanceID string) (bool, erro CORE.Shutdown(EC_OK) } + passwordVariations := passwd.GenPasswordVariations(password) + if iAuth != nil { - if passwd.Check(password, iAuth.Pepper, iAuth.Hash) { - return true, nil + for _, pwd := range passwordVariations { + if passwd.Check(pwd, iAuth.Pepper, iAuth.Hash) { + return true, nil + } } } @@ -723,8 +727,10 @@ func authenticate(authType AuthType, strict bool, instanceID string) (bool, erro } if sAuth != nil { - if passwd.Check(password, sAuth.Pepper, sAuth.Hash) { - return true, nil + for _, pwd := range passwordVariations { + if passwd.Check(pwd, sAuth.Pepper, sAuth.Hash) { + return true, nil + } } } From de02ac12d082b0d0ba1bd86d77c72d74879d1570 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Sun, 10 Sep 2023 23:25:36 +0300 Subject: [PATCH 15/36] Update spec --- common/rds.spec | 23 +++++++++++++++++------ sync/sync.go | 2 +- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/common/rds.spec b/common/rds.spec index b795dc7..2152507 100644 --- a/common/rds.spec +++ b/common/rds.spec @@ -1,6 +1,6 @@ ################################################################################ -%define debug_package %{nil} +%define debug_package %{nil} ################################################################################ @@ -10,7 +10,7 @@ Summary: Redis orchestration tool Name: rds -Version: 1.1.0 +Version: 1.2.0 Release: 0%{?dist} Group: Applications/System License: Apache License, Version 2.0 @@ -21,12 +21,11 @@ Source0: https://source.kaos.st/%{name}/%{name}-%{version}.tar.bz2 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) -BuildRequires: golang >= 1.19 +BuildRequires: golang >= 1.20 Requires: tuned Provides: %{name} = %{version}-%{release} -Provides: %{name}-cli = %{version}-%{release} ################################################################################ @@ -36,9 +35,8 @@ Tool for Redis orchestration. ################################################################################ %package sync - Summary: RDS Sync daemon -Version: 1.1.0 +Version: 1.1.1 Release: 0%{?dist} Group: Applications/System @@ -184,6 +182,19 @@ systemctl daemon-reload &>/dev/null || : ################################################################################ %changelog +* Sun Sep 10 2023 Anton Novojilov - 1.2.0-0 +- [cli] Added 'validate-templates' command for templates validation +- [cli] Added 'backup-create' command for creating RDB snapshots +- [cli] Added 'backup-restore' command for restoring instance data from snapshots +- [cli] Added 'backup-clean' command for deleting RDB snapshots +- [cli] Added 'backup-list' command for listing RDB snapshots +- [cli] Added -R/--raw option for forcing raw output +- [sync] Disable read-only mode for replicas on minion if standby failover is used +- [core] Run all processes with umask 027 +- [cli] Improved properties filtering in 'conf' command +- [cli] Added using of password variations for password auth +- [cli] Improved actions logging + * Tue Aug 22 2023 Anton Novojilov - 1.1.0-0 - Added info about RDS to templates payload - Added instance storage data to templates payload diff --git a/sync/sync.go b/sync/sync.go index e019508..9d08384 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -35,7 +35,7 @@ import ( const ( APP = "RDS Sync" - VER = "1.1.0" + VER = "1.1.1" DESC = "Syncing daemon for RDS" ) From b071e4203f2a14aa7f95c4483fe419f56dc482cf Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Sun, 10 Sep 2023 23:35:57 +0300 Subject: [PATCH 16/36] Update spec --- common/rds.spec | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/rds.spec b/common/rds.spec index 2152507..ebf0e7a 100644 --- a/common/rds.spec +++ b/common/rds.spec @@ -185,11 +185,13 @@ systemctl daemon-reload &>/dev/null || : * Sun Sep 10 2023 Anton Novojilov - 1.2.0-0 - [cli] Added 'validate-templates' command for templates validation - [cli] Added 'backup-create' command for creating RDB snapshots -- [cli] Added 'backup-restore' command for restoring instance data from snapshots +- [cli] Added 'backup-restore' command for restoring instance data from + snapshots - [cli] Added 'backup-clean' command for deleting RDB snapshots - [cli] Added 'backup-list' command for listing RDB snapshots - [cli] Added -R/--raw option for forcing raw output -- [sync] Disable read-only mode for replicas on minion if standby failover is used +- [sync] Disable read-only mode for replicas on minion if standby failover + is used - [core] Run all processes with umask 027 - [cli] Improved properties filtering in 'conf' command - [cli] Added using of password variations for password auth From a814220c5401a9c4526a79d4e13c0c65e414c5e8 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Mon, 11 Sep 2023 13:23:15 +0300 Subject: [PATCH 17/36] Improve help info --- cli/command_help.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cli/command_help.go b/cli/command_help.go index 25a817d..a5d5056 100644 --- a/cli/command_help.go +++ b/cli/command_help.go @@ -136,12 +136,12 @@ func helpCommandCreate() { desc: "Command read user input and create a new instance.", options: []helpInfoArgument{ {getNiceOptions(OPT_TAGS), "List of tags", false}, - {getNiceOptions(OPT_SECURE), "Create instance with ACL", false}, - {getNiceOptions(OPT_DISABLE_SAVES), "Disable saves for created instance", false}, + {getNiceOptions(OPT_SECURE), "Create instance with service ACL", false}, + {getNiceOptions(OPT_DISABLE_SAVES), "Disable saving for created instance", false}, }, examples: []helpInfoExample{ {"", "", "Create new instance"}, - {"", "--disable-saves", "Create new instance with disabled saves"}, + {"", "--disable-saves", "Create new instance with saves disabled"}, {"", "--tags r:important,myapp", "Create new instance with tags"}, }, }.render() @@ -151,7 +151,7 @@ func helpCommandCreate() { func helpCommandDestroy() { helpInfo{ command: COMMAND_RELEASE, - desc: "Command destroy instance associated with the defined ID. Command delete all instance data, logs and configuration files on master and all minions.", + desc: "Command to destroy the instance associated with the given ID. Command deletes all instance data, logs and configuration files on the master and all minions.", arguments: []helpInfoArgument{ {"id", "Instance unique ID", false}, }, @@ -165,7 +165,7 @@ func helpCommandDestroy() { func helpCommandEdit() { helpInfo{ command: COMMAND_EDIT, - desc: "With this command, you can change some info about your instance. At this moment you can change owner, description, and password.", + desc: "This command allows you to change some information about the instance. At the moment you can change the owner, description and password.", arguments: []helpInfoArgument{ {"id", "Instance unique ID", false}, }, @@ -190,7 +190,7 @@ func helpCommandBatchCreate() { info.renderUsage() fmtc.Println("{*}Description{!}\n") - fmtc.Println(` With this command, you can create many instances at once. CSV file must have records in next format: + fmtc.Println(` This command allows you to create many instances at once. The CSV file must contain records in the next format: {m}owner;password;replication-type;auth-password;description{!} @@ -211,7 +211,7 @@ func helpCommandBatchCreate() { func helpCommandBatchEdit() { helpInfo{ command: COMMAND_BATCH_EDIT, - desc: "With this command you can modify metadata for many instances at once.", + desc: "This command allows you to change metadata for many instances at once.", arguments: []helpInfoArgument{ {"id", "Instance ID list", false}, }, @@ -321,7 +321,7 @@ func helpCommandCli() { func helpCommandClients() { helpInfo{ command: COMMAND_CLIENTS, - desc: "Show list of connected clients.", + desc: "Show list of clients connected to the instance.", arguments: []helpInfoArgument{ {"id", "Instance unique ID", false}, {"filter", "Clients filter", true}, @@ -355,7 +355,7 @@ func helpCommandCPU() { func helpCommandInfo() { helpInfo{ command: COMMAND_INFO, - desc: "Show system info about Redis instance.", + desc: "Show info about Redis instance.", arguments: []helpInfoArgument{ {"id", "Instance unique ID", false}, {"section…", "Info section", true}, @@ -456,7 +456,7 @@ func helpCommandConf() { func helpCommandReload() { helpInfo{ command: COMMAND_RELOAD, - desc: "Reload configuration for one or all instances. Use this command if configuration file was updated. Please use this command carefully.", + desc: "Reload the configuration for one or all instances. Use this command if the configuration file has been updated. Use this command with care.", arguments: []helpInfoArgument{ {"id", "Instance unique ID", false}, }, @@ -590,7 +590,7 @@ func helpCommandStatsError() { func helpCommandTop() { helpInfo{ command: COMMAND_TOP, - desc: "Show top for some field available in INFO command output. Without any arguments command show top 10 by memory usage. Also this command can calculate CPU usage (\"cpu\", \"cpu_children\", \"cpu_sys\", \"cpu_user\", \"cpu_sys_children\", \"cpu_user_children\")", + desc: "Show top for any field available in the INFO command output. Command without arguments show top 10 by memory usage. Also this command can calculate CPU usage (\"cpu\", \"cpu_children\", \"cpu_sys\", \"cpu_user\", \"cpu_sys_children\", \"cpu_user_children\")", arguments: []helpInfoArgument{ {"field", "Field name", true}, {"num", "Number of results", true}, @@ -609,7 +609,7 @@ func helpCommandTop() { func helpCommandTopDump() { helpInfo{ command: COMMAND_TOP_DUMP, - desc: "Dump top data to the file. Output file must have .gz extension (all data saved as gzipped JSON file) and must not exist before saving. For output name can be used date control sequences (see 'man date').", + desc: "Dump top data to file. The output file must have a .gz extension (all data saved as a gzipped JSON file) and must not exist before saving. Date control sequences can be used for the output name (see 'man date').", arguments: []helpInfoArgument{ {"file", "Output file", false}, }, @@ -777,7 +777,7 @@ func helpCommandStateRestore() { func helpCommandRegen() { helpInfo{ command: COMMAND_REGEN, - desc: "Regenerate configuration file for one or all instances. Use this command if configuration template was updated. Please use this command carefully.", + desc: "Regenerate the configuration file for one or all instances. Use this command if the configuration template has been updated. Use this command with care.", arguments: []helpInfoArgument{ {"id", "Instance unique ID", false}, }, @@ -792,7 +792,7 @@ func helpCommandRegen() { func helpCommandReplication() { helpInfo{ command: COMMAND_REPLICATION, - desc: "Show info about RDS replication with other nodes in the cluster.", + desc: "Show information about RDS replication with other nodes in the cluster.", options: []helpInfoArgument{ {getNiceOptions(OPT_FORMAT), "Output format (json|text|xml)", false}, }, From 686ecf263d91fef9e016bfd40fc795f80db625e6 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Tue, 12 Sep 2023 15:31:31 +0300 Subject: [PATCH 18/36] [core] Use sync.Map for caching metadata --- core/core.go | 6 ++-- core/metacache.go | 66 ++++++++++++++++++++++++++++++------------- sync/master/master.go | 2 +- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/core/core.go b/core/core.go index b237aa8..4f231bb 100644 --- a/core/core.go +++ b/core/core.go @@ -756,9 +756,9 @@ func GetInstanceMeta(id int) (*InstanceMeta, error) { metaCache = &MetaCache{} } - hit, meta := metaCache.Get(id) + meta, ok := metaCache.Get(id) - if hit { + if ok { return meta, nil } @@ -778,7 +778,7 @@ func GetInstanceMeta(id int) (*InstanceMeta, error) { metaCache.Set(id, meta) - _, meta = metaCache.Get(id) + meta, _ = metaCache.Get(id) return meta, err } diff --git a/core/metacache.go b/core/metacache.go index 55a1182..1edb256 100644 --- a/core/metacache.go +++ b/core/metacache.go @@ -8,16 +8,19 @@ package core // ////////////////////////////////////////////////////////////////////////////////// // import ( + "sync" "time" ) // ////////////////////////////////////////////////////////////////////////////////// // +// MetaCache is instance metadata cache type MetaCache struct { maxCacheTime int64 - items map[int]*MetaCacheItem + items *sync.Map } +// MetaCacheItem contains instance metadata and date of creation type MetaCacheItem struct { meta *InstanceMeta date int64 @@ -29,44 +32,61 @@ type MetaCacheItem struct { func NewMetaCache(maxCacheTime time.Duration) *MetaCache { return &MetaCache{ maxCacheTime: int64(maxCacheTime), - items: make(map[int]*MetaCacheItem), + items: &sync.Map{}, } } // ////////////////////////////////////////////////////////////////////////////////// // -// Set add object to cache +// Set adds metadata to cache func (c *MetaCache) Set(key int, meta *InstanceMeta) { - c.items[key] = &MetaCacheItem{meta, time.Now().UnixNano()} + if c.items == nil { + return + } + + c.items.Store(key, &MetaCacheItem{meta, time.Now().UnixNano()}) + c.clearCache() } -// Get returns meta from cache -func (c *MetaCache) Get(key int) (bool, *InstanceMeta) { - item, hit := c.items[key] +// Get returns meta from cache if exist +func (c *MetaCache) Get(key int) (*InstanceMeta, bool) { + if c.items == nil { + return nil, false + } + + v, ok := c.items.Load(key) + + if ok { + item := v.(*MetaCacheItem) - if hit { if time.Now().UnixNano()-item.date >= c.maxCacheTime { - delete(c.items, key) - return false, nil + c.items.Delete(key) + return nil, false } - return true, c.getClone(item.meta) + return c.getClone(item.meta), true } - return false, nil + return nil, false } // Has checks that we have cached meta for instance func (c *MetaCache) Has(key int) bool { - item, hit := c.items[key] + if c.items == nil { + return false + } + + v, ok := c.items.Load(key) - if !hit { + if !ok { return false } + item := v.(*MetaCacheItem) + if time.Now().UnixNano()-item.date >= c.maxCacheTime { - delete(c.items, key) + c.items.Delete(key) return false } @@ -75,7 +95,11 @@ func (c *MetaCache) Has(key int) bool { // Remove removes item from cache func (c *MetaCache) Remove(key int) { - delete(c.items, key) + if c.items == nil { + return + } + + c.items.Delete(key) } // ////////////////////////////////////////////////////////////////////////////////// // @@ -88,11 +112,15 @@ func (c *MetaCache) clearCache() { now := time.Now().UnixNano() - for key, item := range c.items { + c.items.Range(func(key, value any) bool { + item := value.(*MetaCacheItem) + if now-item.date >= c.maxCacheTime { - delete(c.items, key) + c.items.Delete(key) } - } + + return true + }) } // getClone clones meta diff --git a/sync/master/master.go b/sync/master/master.go index 1388af4..459b1fd 100644 --- a/sync/master/master.go +++ b/sync/master/master.go @@ -753,7 +753,7 @@ func collectInstancesData() []*CORE.InstanceInfo { // appendHeader append header to response func appendHeader(w http.ResponseWriter) { - w.Header().Set("Server", "RSS/"+daemonVersion) + w.Header().Set("Server", "RDS-Sync/"+daemonVersion) w.Header().Set("Content-Type", "application/json") } From bb2e8b1888f5338705e2d0808ab2172058f907a3 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Tue, 12 Sep 2023 17:55:31 +0300 Subject: [PATCH 19/36] Fix daemonize option in templates for Sentinel 7.0 and 7.2 --- common/templates/sentinel/sentinel-7.0.conf | 2 +- common/templates/sentinel/sentinel-7.2.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/templates/sentinel/sentinel-7.0.conf b/common/templates/sentinel/sentinel-7.0.conf index 1617b87..23cd557 100644 --- a/common/templates/sentinel/sentinel-7.0.conf +++ b/common/templates/sentinel/sentinel-7.0.conf @@ -27,7 +27,7 @@ port {{.Port}} # By default Redis Sentinel does not run as a daemon. Use 'yes' if you need it. # Note that Redis will write a pid file in /var/run/redis-sentinel.pid when # daemonized. -daemonize no +daemonize yes # When running daemonized, Redis Sentinel writes a pid file in # /var/run/redis-sentinel.pid by default. You can specify a custom pid file diff --git a/common/templates/sentinel/sentinel-7.2.conf b/common/templates/sentinel/sentinel-7.2.conf index 3b7cfe9..1f77cb3 100644 --- a/common/templates/sentinel/sentinel-7.2.conf +++ b/common/templates/sentinel/sentinel-7.2.conf @@ -27,7 +27,7 @@ port {{.Port}} # By default Redis Sentinel does not run as a daemon. Use 'yes' if you need it. # Note that Redis will write a pid file in /var/run/redis-sentinel.pid when # daemonized. -daemonize no +daemonize yes # When running daemonized, Redis Sentinel writes a pid file in # /var/run/redis-sentinel.pid by default. You can specify a custom pid file From ce71d5a3130402c969f1094884fa247ed31b2165 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Tue, 12 Sep 2023 18:37:38 +0300 Subject: [PATCH 20/36] [cli] Fixed bug with using current user as owner for instances created with 'batch-create' command --- cli/command_batch_create.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/command_batch_create.go b/cli/command_batch_create.go index 60a127f..1631433 100644 --- a/cli/command_batch_create.go +++ b/cli/command_batch_create.go @@ -87,6 +87,7 @@ func BatchCreateCommand(args CommandArgs) int { } meta.Desc = info.Desc + meta.Auth.User = info.Owner meta.Preferencies.ReplicationType = CORE.ReplicationType(info.ReplicationType) meta.Preferencies.IsSaveDisabled = options.GetB(OPT_DISABLE_SAVES) From c0d296232dc36265d95af0e7927a87a008efa182 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Tue, 12 Sep 2023 23:57:51 +0300 Subject: [PATCH 21/36] Improve help output --- README.md | 13 ++++++++----- cli/cli.go | 27 +++++++++++++++++++-------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 29999d0..1d97eff 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ RDS packages do not have Redis as a dependency, so you can install it from any s ``` Usage: rds {options} {command} -Instances commands +Basic commands create Create new Redis instance destroy id Destroy (delete) Redis instance @@ -82,14 +82,17 @@ Instances commands top-dump file Dump top data to file slowlog-get id num Show last entries from slow log slowlog-reset id Clear slow log - backup-create id Create snapshot of RDB file - backup-restore id Restore instance data from snapshot - backup-clean id Delete all backup snapshots - backup-list id List backup snapshots tag-add id tag Add tag to instance tag-remove id tag Remove tag from instance check Check for dead instances +Backup commands + + backup-create id Create snapshot of RDB file + backup-restore id Restore instance data from snapshot + backup-clean id Remove all backup snapshots + backup-list id List backup snapshots + Superuser commands go Generate superuser access credentials diff --git a/cli/cli.go b/cli/cli.go index db01fc8..357cc09 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -866,7 +866,7 @@ func showSmartUsage() { } } - info.AddGroup("Instances commands") + info.AddGroup("Basic commands") if isMaster { info.AddCommand(COMMAND_CREATE, "Create new Redis instance") @@ -900,10 +900,6 @@ func showSmartUsage() { info.AddCommand(COMMAND_TOP_DUMP, "Dump top data to file", "file") info.AddCommand(COMMAND_SLOWLOG_GET, "Show last entries from slow log", "id", "?num") info.AddCommand(COMMAND_SLOWLOG_RESET, "Clear slow log", "id") - info.AddCommand(COMMAND_BACKUP_CREATE, "Create snapshot of RDB file", "id") - info.AddCommand(COMMAND_BACKUP_RESTORE, "Restore instance data from snapshot", "id") - info.AddCommand(COMMAND_BACKUP_CLEAN, "Remove all backup snapshots", "id") - info.AddCommand(COMMAND_BACKUP_LIST, "List backup snapshots", "id") if isMaster { info.AddCommand(COMMAND_TAG_ADD, "Add tag to instance", "id", "tag") @@ -913,6 +909,13 @@ func showSmartUsage() { info.AddCommand(COMMAND_CHECK, "Check for dead instances") } + info.AddGroup("Backup commands") + + info.AddCommand(COMMAND_BACKUP_CREATE, "Create snapshot of RDB file", "id") + info.AddCommand(COMMAND_BACKUP_RESTORE, "Restore instance data from snapshot", "id") + info.AddCommand(COMMAND_BACKUP_CLEAN, "Remove all backup snapshots", "id") + info.AddCommand(COMMAND_BACKUP_LIST, "List backup snapshots", "id") + info.AddGroup("Superuser commands") if isMaster { @@ -1003,7 +1006,8 @@ func genUsage() *usage.Info { info.AppNameColorTag = "{*}" + colorTagApp - info.AddGroup("Instances commands") + info.AddGroup("Basic commands") + info.AddCommand(COMMAND_CREATE, "Create new Redis instance") info.AddCommand(COMMAND_DESTROY, "Destroy (delete) Redis instance", "id") info.AddCommand(COMMAND_EDIT, "Edit metadata for instance", "id") @@ -1031,13 +1035,17 @@ func genUsage() *usage.Info { info.AddCommand(COMMAND_SLOWLOG_RESET, "Clear slow log", "id") info.AddCommand(COMMAND_TAG_ADD, "Add tag to instance", "id", "tag") info.AddCommand(COMMAND_TAG_REMOVE, "Remove tag from instance", "id", "tag") + info.AddCommand(COMMAND_CHECK, "Check for dead instances") + + info.AddGroup("Backup commands") + info.AddCommand(COMMAND_BACKUP_CREATE, "Create snapshot of RDB file", "id") info.AddCommand(COMMAND_BACKUP_RESTORE, "Restore instance data from snapshot", "id") - info.AddCommand(COMMAND_BACKUP_CLEAN, "Delete all backup snapshots", "id") + info.AddCommand(COMMAND_BACKUP_CLEAN, "Remove all backup snapshots", "id") info.AddCommand(COMMAND_BACKUP_LIST, "List backup snapshots", "id") - info.AddCommand(COMMAND_CHECK, "Check for dead instances") info.AddGroup("Superuser commands") + info.AddCommand(COMMAND_GO, "Generate superuser access credentials") info.AddCommand(COMMAND_BATCH_CREATE, "Create many instances at once", "csv-file") info.AddCommand(COMMAND_BATCH_EDIT, "Edit many instances at once", "id") @@ -1051,10 +1059,12 @@ func genUsage() *usage.Info { info.AddCommand(COMMAND_MAINTENANCE, "Enable or disable maintenance mode", "flag") info.AddGroup("Replication commands") + info.AddCommand(COMMAND_REPLICATION, "Show replication info") info.AddCommand(COMMAND_REPLICATION_ROLE_SET, "Reconfigure node after changing the role") info.AddGroup("Sentinel commands") + info.AddCommand(COMMAND_SENTINEL_START, "Start Redis Sentinel daemon") info.AddCommand(COMMAND_SENTINEL_STOP, "Stop Redis Sentinel daemon") info.AddCommand(COMMAND_SENTINEL_STATUS, "Show status of Redis Sentinel daemon") @@ -1065,6 +1075,7 @@ func genUsage() *usage.Info { info.AddCommand(COMMAND_SENTINEL_SWITCH, "Switch instance to master role", "id") info.AddGroup("Common commands") + info.AddCommand(COMMAND_HELP, "Show command usage info", "command") info.AddCommand(COMMAND_SETTINGS, "Show settings from global configuration file", "?section…") info.AddCommand(COMMAND_GEN_TOKEN, "Generate authentication token for sync daemon") From 7846724896beab5fa8f03764ee0ce7f3cded9752 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 13 Sep 2023 00:08:07 +0300 Subject: [PATCH 22/36] Update spec --- common/rds.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/common/rds.spec b/common/rds.spec index ebf0e7a..df106c9 100644 --- a/common/rds.spec +++ b/common/rds.spec @@ -193,6 +193,7 @@ systemctl daemon-reload &>/dev/null || : - [sync] Disable read-only mode for replicas on minion if standby failover is used - [core] Run all processes with umask 027 +- [core] Use sync.Map for caching metadata - [cli] Improved properties filtering in 'conf' command - [cli] Added using of password variations for password auth - [cli] Improved actions logging From 85a8e2bbc994970e4c2946f2b40035a6b78e395f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 03:54:02 +0000 Subject: [PATCH 23/36] Bump docker/login-action from 2 to 3 Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f5f9bc..e2d4b76 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: uses: actions/checkout@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} From d67067e8db27a86acd69b33a2ccf9f130a8e7716 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 13 Sep 2023 11:44:31 +0300 Subject: [PATCH 24/36] [cli] Fix raw output from 'top' command for long numbers --- cli/command_top.go | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/cli/command_top.go b/cli/command_top.go index 583f523..73a8e2e 100644 --- a/cli/command_top.go +++ b/cli/command_top.go @@ -31,11 +31,12 @@ import ( // ////////////////////////////////////////////////////////////////////////////////// // type topItem struct { - ID int - Value float64 + ID int + Value float64 + IsFloat bool } -type topItems []*topItem +type topItems []topItem type topDump struct { Data []*topDumpItem `json:"data"` @@ -199,10 +200,10 @@ func collectTopInfo(field string) (topItems, error) { switch field { case "keys": - items = append(items, &topItem{id, float64(info.Keyspace.Keys())}) + items = append(items, topItem{id, float64(info.Keyspace.Keys()), false}) continue case "expires": - items = append(items, &topItem{id, float64(info.Keyspace.Expires())}) + items = append(items, topItem{id, float64(info.Keyspace.Expires()), false}) continue } @@ -221,7 +222,7 @@ func collectTopInfo(field string) (topItems, error) { return nil, fmt.Errorf("Field \"%s\" has an unsupported type", field) } - items = append(items, &topItem{id, value}) + items = append(items, topItem{id, value, strings.Contains(str, ".")}) } return items, nil @@ -272,17 +273,17 @@ func collectTopCPUInfo(field string) (topItems, error) { switch field { case "cpu_sys": - items = append(items, &topItem{id, usage[0]}) + items = append(items, topItem{id, usage[0], true}) case "cpu_user": - items = append(items, &topItem{id, usage[1]}) + items = append(items, topItem{id, usage[1], true}) case "cpu_sys_children": - items = append(items, &topItem{id, usage[2]}) + items = append(items, topItem{id, usage[2], true}) case "cpu_user_children": - items = append(items, &topItem{id, usage[3]}) + items = append(items, topItem{id, usage[3], true}) case "cpu": - items = append(items, &topItem{id, usage[0] + usage[1]}) + items = append(items, topItem{id, usage[0] + usage[1], true}) case "cpu_children": - items = append(items, &topItem{id, usage[2] + usage[3]}) + items = append(items, topItem{id, usage[2] + usage[3], true}) } } @@ -342,7 +343,11 @@ func printTopInfo(items topItems, resultNum int, diff bool) { } } else { - fmtc.Printf("%d %v\n", item.ID, items[i].Value) + if items[i].IsFloat { + fmtc.Printf("%d %f\n", item.ID, items[i].Value) + } else { + fmtc.Printf("%d %.0f\n", item.ID, items[i].Value) + } } if i == resultNum-1 { @@ -409,7 +414,7 @@ func diffTopData(field string, curTop topItems, dumpData []*topDumpItem) (topIte return nil, fmt.Errorf("Field \"%s\" has an unsupported type", field) } - result = append(result, &topItem{c.ID, diff}) + result = append(result, topItem{c.ID, diff, strings.Contains(v, ".")}) } return result, nil From 55e4538611a5dd4acaedb851a2aa49e29e62d0a8 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 13 Sep 2023 11:44:59 +0300 Subject: [PATCH 25/36] Update spec --- common/rds.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/common/rds.spec b/common/rds.spec index df106c9..55b7118 100644 --- a/common/rds.spec +++ b/common/rds.spec @@ -183,6 +183,7 @@ systemctl daemon-reload &>/dev/null || : %changelog * Sun Sep 10 2023 Anton Novojilov - 1.2.0-0 +- [cli] Fixed raw output from 'top' command for long numbers - [cli] Added 'validate-templates' command for templates validation - [cli] Added 'backup-create' command for creating RDB snapshots - [cli] Added 'backup-restore' command for restoring instance data from From 43ac6bf253c70dc9cd4c017d28b4fb0599bc5bc7 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 13 Sep 2023 13:58:57 +0300 Subject: [PATCH 26/36] [cli] Fix command execution with 'cli' --- cli/command_cli.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cli/command_cli.go b/cli/command_cli.go index b5286f6..35af7cb 100644 --- a/cli/command_cli.go +++ b/cli/command_cli.go @@ -81,7 +81,11 @@ func CliCommand(args CommandArgs) int { cliCfg.Password = meta.Preferencies.ServicePassword } - err = RC.RunRedisCli(cliCfg) + if len(args) == 1 { + err = RC.RunRedisCli(cliCfg) + } else { + err = RC.ExecRedisCmd(cliCfg) + } if err != nil { terminal.Error(err.Error()) From 563b8ec96fd66e155ab16c47061deee31ba89d56 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 13 Sep 2023 14:04:21 +0300 Subject: [PATCH 27/36] [cli] Fix password check using password variations --- cli/cli.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cli/cli.go b/cli/cli.go index 357cc09..5acb130 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -713,6 +713,10 @@ func authenticate(authType AuthType, strict bool, instanceID string) (bool, erro passwordVariations := passwd.GenPasswordVariations(password) if iAuth != nil { + if passwd.Check(password, iAuth.Pepper, iAuth.Hash) { + return true, nil + } + for _, pwd := range passwordVariations { if passwd.Check(pwd, iAuth.Pepper, iAuth.Hash) { return true, nil @@ -727,6 +731,10 @@ func authenticate(authType AuthType, strict bool, instanceID string) (bool, erro } if sAuth != nil { + if passwd.Check(password, sAuth.Pepper, sAuth.Hash) { + return true, nil + } + for _, pwd := range passwordVariations { if passwd.Check(pwd, sAuth.Pepper, sAuth.Hash) { return true, nil From 8b5dc3d11e3ce58d2ee8e4d7a18ec96795da77e0 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 13 Sep 2023 14:04:57 +0300 Subject: [PATCH 28/36] [cli] Fix copy-paste error message --- cli/command_stats_error.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/command_stats_error.go b/cli/command_stats_error.go index d1c2bbd..e990b0a 100644 --- a/cli/command_stats_error.go +++ b/cli/command_stats_error.go @@ -58,7 +58,7 @@ func printErrorStatsInfo(info *REDIS.Info) { section := info.Sections["errorstats"] if section == nil || len(section.Fields) == 0 { - terminal.Warn("There is no info about latency") + terminal.Warn("There is no info about errors") return } From c35b7db6b77abad6cbe01f0be90b1303b8794be4 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 13 Sep 2023 14:16:57 +0300 Subject: [PATCH 29/36] Update spec --- common/rds.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/rds.spec b/common/rds.spec index 55b7118..25c518a 100644 --- a/common/rds.spec +++ b/common/rds.spec @@ -183,7 +183,6 @@ systemctl daemon-reload &>/dev/null || : %changelog * Sun Sep 10 2023 Anton Novojilov - 1.2.0-0 -- [cli] Fixed raw output from 'top' command for long numbers - [cli] Added 'validate-templates' command for templates validation - [cli] Added 'backup-create' command for creating RDB snapshots - [cli] Added 'backup-restore' command for restoring instance data from @@ -198,6 +197,10 @@ systemctl daemon-reload &>/dev/null || : - [cli] Improved properties filtering in 'conf' command - [cli] Added using of password variations for password auth - [cli] Improved actions logging +- [cli] Fixed raw output from 'top' command for long numbers +- [cli] Fixed command execution with 'cli' +- [cli] Fixed password check using password variations +- [cli] Fix password check using password variations * Tue Aug 22 2023 Anton Novojilov - 1.1.0-0 - Added info about RDS to templates payload From 02912a139154cdce83b3cfeacc942682fa61b41f Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 13 Sep 2023 14:28:40 +0300 Subject: [PATCH 30/36] [cli] Fix command execution with 'cli' --- redis/client/redis_cli.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/redis/client/redis_cli.go b/redis/client/redis_cli.go index 1c3a2b0..c84f5b8 100644 --- a/redis/client/redis_cli.go +++ b/redis/client/redis_cli.go @@ -168,7 +168,7 @@ func RunRedisCli(cfg *Config) error { // ////////////////////////////////////////////////////////////////////////////////// // -// execCommand exec one command +// execCommand execs one command func execCommand(cfg *Config) error { client := getClient(cfg.Port, time.Second*time.Duration(cfg.Timeout)) @@ -180,6 +180,8 @@ func execCommand(cfg *Config) error { defer client.Close() + configureClient(client, cfg) + var resp *redy.Resp switch len(cfg.Command) { From 9b8bc77081ea1f051cfdd35ffbfb45319479efe8 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 13 Sep 2023 16:27:26 +0300 Subject: [PATCH 31/36] [cli] Improve instance listing filtering --- cli/command_help.go | 2 +- cli/command_list.go | 6 ++++-- cli/common.go | 27 ++++++++++++++++----------- common/rds.spec | 1 + 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/cli/command_help.go b/cli/command_help.go index a5d5056..6756e6c 100644 --- a/cli/command_help.go +++ b/cli/command_help.go @@ -498,9 +498,9 @@ func helpCommandList() { fmtc.Printf(" {b}%-10s{!} %s\n", "saving", "Saving instances") fmtc.Printf(" {b}%-10s{!} %s\n", "loading", "Loading instances") fmtc.Printf(" {b}%-10s{!} %s\n", "outdated", "Instances which require restart for update") + fmtc.Printf(" {b}%-10s{!} %s\n", "orphan", "Instances owned by deleted users") fmtc.Printf(" {b}%-10s{!} %s\n", "standby", "Instances with standby replication") fmtc.Printf(" {b}%-10s{!} %s\n", "replica", "Instances with real replicas") - fmtc.Printf(" {b}%-10s{!} %s\n", "sentinel", "Instances with enabled Sentinel support") fmtc.Printf(" {b}%-10s{!} %s\n", "secure", "Instances with enabled authentication") fmtc.Printf(" {b}%-10s{!} %s\n", "{username}", "Instances owned by given user") fmtc.Printf(" {b}%-10s{!} %s\n", "{tag}", "Instances tagged by given tag") diff --git a/cli/command_list.go b/cli/command_list.go index eaae06b..2fda53a 100644 --- a/cli/command_list.go +++ b/cli/command_list.go @@ -124,6 +124,8 @@ func isFilterFit(filter []string, state CORE.State, meta *CORE.InstanceMeta) boo fit = state.IsWorks() && state.IsSaving() case "loading", "load": fit = state.IsWorks() && state.IsLoading() + case "orphan": + fit = isInstanceOwnerExist(meta.Auth.User) == false case "outdated": currentRedisVer, _ := CORE.GetRedisVersion() if state.IsStopped() { @@ -131,9 +133,9 @@ func isFilterFit(filter []string, state CORE.State, meta *CORE.InstanceMeta) boo } else if currentRedisVer.String() != "" && meta.Compatible != "" { fit = currentRedisVer.String() != meta.Compatible } - case "standby", "duplicate": + case "standby": fit = meta.Preferencies.ReplicationType == CORE.REPL_TYPE_STANDBY - case "replica", "slave": + case "replica": fit = meta.Preferencies.ReplicationType == CORE.REPL_TYPE_REPLICA case "secure": fit = meta.Preferencies.ServicePassword != "" diff --git a/cli/common.go b/cli/common.go index cd6b7c1..1375ec6 100644 --- a/cli/common.go +++ b/cli/common.go @@ -302,17 +302,7 @@ func getInstanceOwnerWithColor(meta *CORE.InstanceMeta, before bool) string { owner := meta.Auth.User - if userExistenceCache == nil { - userExistenceCache = make(map[string]bool) - } - - _, found := userExistenceCache[owner] - - if !found { - userExistenceCache[owner] = system.IsUserExist(owner) - } - - if userExistenceCache[owner] { + if isInstanceOwnerExist(owner) { if CORE.User.RealName == owner { switch before { case true: @@ -328,6 +318,21 @@ func getInstanceOwnerWithColor(meta *CORE.InstanceMeta, before bool) string { return "{s-}" + owner + "{!}" } +// isInstanceOwnerExist returns true if instance owner user exist on the system +func isInstanceOwnerExist(owner string) bool { + if userExistenceCache == nil { + userExistenceCache = make(map[string]bool) + } + + _, found := userExistenceCache[owner] + + if !found { + userExistenceCache[owner] = system.IsUserExist(owner) + } + + return userExistenceCache[owner] +} + // getInstanceDescWithTags returns instance description with rendered tags func getInstanceDescWithTags(meta *CORE.InstanceMeta) string { if meta == nil { diff --git a/common/rds.spec b/common/rds.spec index 25c518a..3bfd7ed 100644 --- a/common/rds.spec +++ b/common/rds.spec @@ -196,6 +196,7 @@ systemctl daemon-reload &>/dev/null || : - [core] Use sync.Map for caching metadata - [cli] Improved properties filtering in 'conf' command - [cli] Added using of password variations for password auth +- [cli] Improved instance listing filtering - [cli] Improved actions logging - [cli] Fixed raw output from 'top' command for long numbers - [cli] Fixed command execution with 'cli' From 0c5db21e7af2e05fc3bfccb726c0a9206264666c Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Wed, 13 Sep 2023 18:09:43 +0300 Subject: [PATCH 32/36] Update spec --- common/rds.spec | 1 - 1 file changed, 1 deletion(-) diff --git a/common/rds.spec b/common/rds.spec index 3bfd7ed..042f967 100644 --- a/common/rds.spec +++ b/common/rds.spec @@ -201,7 +201,6 @@ systemctl daemon-reload &>/dev/null || : - [cli] Fixed raw output from 'top' command for long numbers - [cli] Fixed command execution with 'cli' - [cli] Fixed password check using password variations -- [cli] Fix password check using password variations * Tue Aug 22 2023 Anton Novojilov - 1.1.0-0 - Added info about RDS to templates payload From 1685bf13c359f53a84aaf6e0ddae669ed057204c Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Thu, 14 Sep 2023 00:14:21 +0300 Subject: [PATCH 33/36] [sync] 'max-init-sync-wait' depricated --- common/rds.knf | 4 ---- common/rds.spec | 1 + core/core.go | 5 ----- sync/master/master.go | 13 ++++--------- 4 files changed, 5 insertions(+), 18 deletions(-) diff --git a/common/rds.knf b/common/rds.knf index 8da2a27..a506d9f 100644 --- a/common/rds.knf +++ b/common/rds.knf @@ -152,10 +152,6 @@ # restart-all) to minions always-propagate: true - # Max time (in seconds) which given to minions for initial state sync with - # master (time between fetch and pull command) - max-init-sync-wait: 3600 - # Max time (in seconds) which given to Redis replicas for syncing with masters # Increase this value if you have big instances (20+ Gb) or slow connection # between nodes diff --git a/common/rds.spec b/common/rds.spec index 042f967..a8c92e4 100644 --- a/common/rds.spec +++ b/common/rds.spec @@ -198,6 +198,7 @@ systemctl daemon-reload &>/dev/null || : - [cli] Added using of password variations for password auth - [cli] Improved instance listing filtering - [cli] Improved actions logging +- [sync] 'max-init-sync-wait' depricated - [cli] Fixed raw output from 'top' command for long numbers - [cli] Fixed command execution with 'cli' - [cli] Fixed password check using password variations diff --git a/core/core.go b/core/core.go index 4f231bb..815bdec 100644 --- a/core/core.go +++ b/core/core.go @@ -84,8 +84,6 @@ const ( MAX_TAGS = 3 MIN_SYNC_WAIT = 60 // 1 Min MAX_SYNC_WAIT = 3 * 60 * 60 // 3 Hours - MIN_INIT_SYNC_WAIT = 5 * 60 // 5 min - MAX_INIT_SYNC_WAIT = 6 * 60 * 60 // 6 Hours MAX_FULL_START_DELAY = 30 * 60 // 30 Min MAX_SWITCH_WAIT = 15 * 60 // 15 Min TOKEN_LENGTH = 64 @@ -168,7 +166,6 @@ const ( REPLICATION_ALLOW_REPLICAS = "replication:allow-replicas" REPLICATION_ALLOW_COMMANDS = "replication:allow-commands" REPLICATION_ALWAYS_PROPAGATE = "replication:always-propagate" - REPLICATION_MAX_INIT_SYNC_WAIT = "replication:max-init-sync-wait" REPLICATION_MAX_SYNC_WAIT = "replication:max-sync-wait" REPLICATION_INIT_SYNC_DELAY = "replication:init-sync-delay" @@ -2581,8 +2578,6 @@ func validateConfig(c *knf.Config) []error { &knf.Validator{REPLICATION_AUTH_TOKEN, knfv.NotLen, TOKEN_LENGTH}, &knf.Validator{REPLICATION_MAX_SYNC_WAIT, knfv.Less, MIN_SYNC_WAIT}, &knf.Validator{REPLICATION_MAX_SYNC_WAIT, knfv.Greater, MAX_SYNC_WAIT}, - &knf.Validator{REPLICATION_MAX_INIT_SYNC_WAIT, knfv.Less, MIN_INIT_SYNC_WAIT}, - &knf.Validator{REPLICATION_MAX_INIT_SYNC_WAIT, knfv.Greater, MAX_INIT_SYNC_WAIT}, &knf.Validator{REPLICATION_FAILOVER_METHOD, knfv.NotContains, []string{ string(FAILOVER_METHOD_STANDBY), string(FAILOVER_METHOD_SENTINEL), }}, diff --git a/sync/master/master.go b/sync/master/master.go index 459b1fd..cbbde92 100644 --- a/sync/master/master.go +++ b/sync/master/master.go @@ -504,8 +504,9 @@ func fetchHandler(w http.ResponseWriter, r *http.Request) { log.Error("Can't encode response: %v", err) } - maxInitTimeDur := time.Duration(CORE.Config.GetI(CORE.REPLICATION_MAX_INIT_SYNC_WAIT)) - deadline := time.Now().Add(time.Second * maxInitTimeDur) + maxSyncWait := CORE.Config.GetD(CORE.REPLICATION_MAX_SYNC_WAIT) + maxInitTimeDur := maxSyncWait * time.Duration(len(fetchResponse.Instances)) + deadline := time.Now().Add(maxInitTimeDur) log.Info( "Client with ID %s started initial synchronization process (deadline: %s)", @@ -870,13 +871,7 @@ func getClientState(now int64, client *ClientInfo) API.ClientState { timeDiff := now - client.LastSeen if client.Syncing { - maxInitSyncWait := int64(CORE.Config.GetI(CORE.REPLICATION_MAX_INIT_SYNC_WAIT)) - - if timeDiff <= maxInitSyncWait*1000000000 { - return API.STATE_SYNCING - } else { - return API.STATE_DOWN - } + return API.STATE_SYNCING } switch { From eceff2d317c7e850f39da02cc75734199c534aca Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Thu, 14 Sep 2023 14:20:06 +0300 Subject: [PATCH 34/36] [cli] Fix handling MONITOR in 'cli' --- cli/command_cli.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/command_cli.go b/cli/command_cli.go index 35af7cb..fbe11a6 100644 --- a/cli/command_cli.go +++ b/cli/command_cli.go @@ -9,6 +9,7 @@ package cli import ( "fmt" + "strings" "github.com/essentialkaos/ek/v12/options" "github.com/essentialkaos/ek/v12/terminal" @@ -55,7 +56,7 @@ func CliCommand(args CommandArgs) int { } } - if args.Has(1) || args.Get(1) == "MONITOR" { + if args.Has(1) || strings.ToUpper(args.Get(1)) == "MONITOR" { ops, err := getCurrentInstanceTraffic(id) if err != nil { From 3ba32c41c7e19123d1c66b4fcedacb379e8a43f3 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Thu, 14 Sep 2023 23:35:48 +0300 Subject: [PATCH 35/36] Dependencies update --- go.mod | 6 +++--- go.sum | 12 ++++++------ sync/master/master.go | 2 +- sync/minion/minion.go | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index c0ff9c6..e3d3d68 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,12 @@ go 1.20 require ( github.com/essentialkaos/depsy v1.1.0 - github.com/essentialkaos/ek/v12 v12.75.1 + github.com/essentialkaos/ek/v12 v12.76.0 github.com/essentialkaos/go-linenoise/v3 v3.4.0 github.com/essentialkaos/redy/v4 v4.4.0 ) require ( - golang.org/x/crypto v0.12.0 // indirect - golang.org/x/sys v0.11.0 // indirect + golang.org/x/crypto v0.13.0 // indirect + golang.org/x/sys v0.12.0 // indirect ) diff --git a/go.sum b/go.sum index 1ee01f4..d0ff026 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/essentialkaos/check v1.4.0 h1:kWdFxu9odCxUqo1NNFNJmguGrDHgwi3A8daXX1nkuKk= github.com/essentialkaos/depsy v1.1.0 h1:U6dp687UkQwXlZU17Hg2KMxbp3nfZAoZ8duaeUFYvJI= github.com/essentialkaos/depsy v1.1.0/go.mod h1:kpiTAV17dyByVnrbNaMcZt2jRwvuXClUYOzpyJQwtG8= -github.com/essentialkaos/ek/v12 v12.75.1 h1:HE8/uWED+QgyT6HIcRULqjMYWKhqQ1rkfa/ozcva1eQ= -github.com/essentialkaos/ek/v12 v12.75.1/go.mod h1:juDcZWOWaj1QmYShZkT9RzdqJ3n0tmeP/iq4sw5fQF0= +github.com/essentialkaos/ek/v12 v12.76.0 h1:TDm6q4NFNpiCeozcLigf9V2Z0Pzib7NtPH7F2dsfcmw= +github.com/essentialkaos/ek/v12 v12.76.0/go.mod h1:S9/XSKhEAdylL3PF8GAnUeKKyd92VrDGR4YGacHfz0c= github.com/essentialkaos/go-linenoise/v3 v3.4.0 h1:g72w8x+/HIwOMBVvNaPYp+wMWVHrYZwzFAF7OfZR5Ts= github.com/essentialkaos/go-linenoise/v3 v3.4.0/go.mod h1:t1kNLY2bSMQCy1JXOefD2BDLs/TTPMtTv3DFNV5uDSI= github.com/essentialkaos/redy/v4 v4.4.0 h1:6r6AkZiDkFWPqnvl0M+u7mcaaWQaeeiZOoLqLAMcnzQ= @@ -10,7 +10,7 @@ github.com/essentialkaos/redy/v4 v4.4.0/go.mod h1:0TUOQZGzhRmZD13sCVUtYUYCa3Xd8v github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/sync/master/master.go b/sync/master/master.go index cbbde92..f815b62 100644 --- a/sync/master/master.go +++ b/sync/master/master.go @@ -504,7 +504,7 @@ func fetchHandler(w http.ResponseWriter, r *http.Request) { log.Error("Can't encode response: %v", err) } - maxSyncWait := CORE.Config.GetD(CORE.REPLICATION_MAX_SYNC_WAIT) + maxSyncWait := CORE.Config.GetD(CORE.REPLICATION_MAX_SYNC_WAIT, time.Second) maxInitTimeDur := maxSyncWait * time.Duration(len(fetchResponse.Instances)) deadline := time.Now().Add(maxInitTimeDur) diff --git a/sync/minion/minion.go b/sync/minion/minion.go index 3583282..6befda8 100644 --- a/sync/minion/minion.go +++ b/sync/minion/minion.go @@ -991,7 +991,7 @@ func syncBlocker(id int) { log.Info("(%3d) Starting sync with master instance…", id) - time.Sleep(CORE.Config.GetD(CORE.REPLICATION_INIT_SYNC_DELAY, 3*time.Second)) + time.Sleep(CORE.Config.GetD(CORE.REPLICATION_INIT_SYNC_DELAY, time.Second, 3*time.Second)) syncingWaitLoop(id) } @@ -999,7 +999,7 @@ func syncBlocker(id int) { // syncingWaitLoop blocks main sync process till syncing will be completed func syncingWaitLoop(id int) { start := time.Now().Unix() - maxWait := CORE.Config.GetD(CORE.REPLICATION_MAX_SYNC_WAIT) + maxWait := CORE.Config.GetD(CORE.REPLICATION_MAX_SYNC_WAIT, time.Second) deadline := time.Now().Add(maxWait) log.Info( From 1d0c98a9379bb4a362b2fe3a9413ad83c7be4e18 Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Fri, 15 Sep 2023 00:22:39 +0300 Subject: [PATCH 36/36] Dependencies update --- go.mod | 2 +- go.sum | 4 ++-- sync/master/master.go | 3 ++- sync/minion/minion.go | 5 +++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index e3d3d68..3219f68 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/essentialkaos/depsy v1.1.0 - github.com/essentialkaos/ek/v12 v12.76.0 + github.com/essentialkaos/ek/v12 v12.76.1 github.com/essentialkaos/go-linenoise/v3 v3.4.0 github.com/essentialkaos/redy/v4 v4.4.0 ) diff --git a/go.sum b/go.sum index d0ff026..eb465f9 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/essentialkaos/check v1.4.0 h1:kWdFxu9odCxUqo1NNFNJmguGrDHgwi3A8daXX1nkuKk= github.com/essentialkaos/depsy v1.1.0 h1:U6dp687UkQwXlZU17Hg2KMxbp3nfZAoZ8duaeUFYvJI= github.com/essentialkaos/depsy v1.1.0/go.mod h1:kpiTAV17dyByVnrbNaMcZt2jRwvuXClUYOzpyJQwtG8= -github.com/essentialkaos/ek/v12 v12.76.0 h1:TDm6q4NFNpiCeozcLigf9V2Z0Pzib7NtPH7F2dsfcmw= -github.com/essentialkaos/ek/v12 v12.76.0/go.mod h1:S9/XSKhEAdylL3PF8GAnUeKKyd92VrDGR4YGacHfz0c= +github.com/essentialkaos/ek/v12 v12.76.1 h1:WwmvjZQtAub2+89tzMHZcJELtIDksDVIXrMFfdg0G1w= +github.com/essentialkaos/ek/v12 v12.76.1/go.mod h1:S9/XSKhEAdylL3PF8GAnUeKKyd92VrDGR4YGacHfz0c= github.com/essentialkaos/go-linenoise/v3 v3.4.0 h1:g72w8x+/HIwOMBVvNaPYp+wMWVHrYZwzFAF7OfZR5Ts= github.com/essentialkaos/go-linenoise/v3 v3.4.0/go.mod h1:t1kNLY2bSMQCy1JXOefD2BDLs/TTPMtTv3DFNV5uDSI= github.com/essentialkaos/redy/v4 v4.4.0 h1:6r6AkZiDkFWPqnvl0M+u7mcaaWQaeeiZOoLqLAMcnzQ= diff --git a/sync/master/master.go b/sync/master/master.go index f815b62..ab52d69 100644 --- a/sync/master/master.go +++ b/sync/master/master.go @@ -22,6 +22,7 @@ import ( "github.com/essentialkaos/ek/v12/fsutil" "github.com/essentialkaos/ek/v12/httputil" + "github.com/essentialkaos/ek/v12/knf" "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/mathutil" "github.com/essentialkaos/ek/v12/netutil" @@ -504,7 +505,7 @@ func fetchHandler(w http.ResponseWriter, r *http.Request) { log.Error("Can't encode response: %v", err) } - maxSyncWait := CORE.Config.GetD(CORE.REPLICATION_MAX_SYNC_WAIT, time.Second) + maxSyncWait := CORE.Config.GetD(CORE.REPLICATION_MAX_SYNC_WAIT, knf.Second) maxInitTimeDur := maxSyncWait * time.Duration(len(fetchResponse.Instances)) deadline := time.Now().Add(maxInitTimeDur) diff --git a/sync/minion/minion.go b/sync/minion/minion.go index 6befda8..99f1dfc 100644 --- a/sync/minion/minion.go +++ b/sync/minion/minion.go @@ -16,6 +16,7 @@ import ( "time" "github.com/essentialkaos/ek/v12/fmtutil" + "github.com/essentialkaos/ek/v12/knf" "github.com/essentialkaos/ek/v12/log" "github.com/essentialkaos/ek/v12/mathutil" "github.com/essentialkaos/ek/v12/pluralize" @@ -991,7 +992,7 @@ func syncBlocker(id int) { log.Info("(%3d) Starting sync with master instance…", id) - time.Sleep(CORE.Config.GetD(CORE.REPLICATION_INIT_SYNC_DELAY, time.Second, 3*time.Second)) + time.Sleep(CORE.Config.GetD(CORE.REPLICATION_INIT_SYNC_DELAY, knf.Second, 3*time.Second)) syncingWaitLoop(id) } @@ -999,7 +1000,7 @@ func syncBlocker(id int) { // syncingWaitLoop blocks main sync process till syncing will be completed func syncingWaitLoop(id int) { start := time.Now().Unix() - maxWait := CORE.Config.GetD(CORE.REPLICATION_MAX_SYNC_WAIT, time.Second) + maxWait := CORE.Config.GetD(CORE.REPLICATION_MAX_SYNC_WAIT, knf.Second) deadline := time.Now().Add(maxWait) log.Info(