From 9ce7fa30a41e507d5608efea8fd6760329280da1 Mon Sep 17 00:00:00 2001 From: withsoull Date: Mon, 23 Dec 2024 14:36:31 +0300 Subject: [PATCH 01/11] Add new field `IsLoadingBinlog` to NodeState --- internal/app/app.go | 2 ++ internal/app/data.go | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/internal/app/app.go b/internal/app/app.go index 9243597..d75bc1f 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -166,10 +166,12 @@ func (app *App) removeMaintenanceFile() { // separate goroutine performing health checks func (app *App) healthChecker(ctx context.Context) { ticker := time.NewTicker(app.config.HealthCheckInterval) + var oldBinLogPos string for { select { case <-ticker.C: hc := app.getLocalNodeState() + oldBinLogPos = hc.UpdateBinlogStatus(oldBinLogPos) app.logger.Infof("healthcheck: %v", hc) err := app.dcs.SetEphemeral(dcs.JoinPath(pathHealthPrefix, app.config.Hostname), hc) if err != nil { diff --git a/internal/app/data.go b/internal/app/data.go index 941fbfd..ccf7f5b 100644 --- a/internal/app/data.go +++ b/internal/app/data.go @@ -91,6 +91,7 @@ type NodeState struct { IsOffline bool `json:"is_offline"` IsCascade bool `json:"is_cascade"` IsFileSystemReadonly bool `json:"is_file_system_readonly"` + IsLoadingBinlog bool `json:"is_loading_binlog"` Error string `json:"error"` DiskState *DiskState `json:"disk_state"` DaemonState *DaemonState `json:"daemon_state"` @@ -144,6 +145,20 @@ func (ns *NodeState) CalcGTIDDiffWithMaster() (string, error) { return gtids.GTIDDiff(replicaGTID, sourceGTID) } +func (ns *NodeState) UpdateBinlogStatus(oldBinloPos string) (newBinlogPos string) { + if ns.SlaveState != nil { + newBinlogPos = fmt.Sprintf("%s%019d", ns.SlaveState.MasterLogFile, ns.SlaveState.MasterLogPos) + + if newBinlogPos > oldBinloPos { + ns.IsLoadingBinlog = true + } else { + ns.IsLoadingBinlog = false + } + } + + return +} + func (ns *NodeState) String() string { ping := "ok" if !ns.PingOk { From 4eb9345454f32de40936ba2d9af5bd2db06617a9 Mon Sep 17 00:00:00 2001 From: withsoull Date: Mon, 23 Dec 2024 14:39:34 +0300 Subject: [PATCH 02/11] Add unit and feature tests for new flag `IsLoadingBinlog` of nodes - Some reveiw of old unit tests --- internal/app/data_test.go | 57 +++++++++++++++++++++++---- tests/features/binlog_loading.feature | 31 +++++++++++++++ 2 files changed, 80 insertions(+), 8 deletions(-) create mode 100644 tests/features/binlog_loading.feature diff --git a/internal/app/data_test.go b/internal/app/data_test.go index 096277a..155611a 100644 --- a/internal/app/data_test.go +++ b/internal/app/data_test.go @@ -3,6 +3,8 @@ package app import ( "fmt" "testing" + + "github.com/stretchr/testify/require" ) func TestStringerWorksOnNodeState(t *testing.T) { @@ -19,15 +21,21 @@ func TestStringerWorksOnNodeState(t *testing.T) { ns.SlaveState.ExecutedGtidSet = "6DBC0B04-4B09-43DC-86CC-9AF852DED919:1-40" nsStr = fmt.Sprintf("%v", ns) - if nsStr != "" { - t.Errorf("%s", ns) - } + + require.Equal( + t, + "", + nsStr, + ) ns.ShowOnlyGTIDDiff = true nsStr = fmt.Sprintf("%v", ns) - if nsStr != "" { - t.Errorf("%s", ns) - } + + require.Equal( + t, + "", + nsStr, + ) } func TestStringerWorksOnNodeStateMap(t *testing.T) { @@ -37,7 +45,40 @@ func TestStringerWorksOnNodeStateMap(t *testing.T) { m["c"] = &NodeState{} mStr := fmt.Sprintf("%v", m) - if mStr != "map[a: b: c:]" { - t.Errorf("%s", mStr) + + require.Equal( + t, + "map[a: b: c:]", + mStr, + ) +} + +func newMockNodeState() *NodeState { + return &NodeState{ + SlaveState: &SlaveState{ + MasterLogFile: "test_master_log_file", + MasterLogPos: 2, + }, } } + +func TestUpdateBinlogWithChanges(t *testing.T) { + oldBinlogPostion := "test_master_log_file0000000000000000001" + ns := newMockNodeState() + + newBinlogPos := ns.UpdateBinlogStatus(oldBinlogPostion) + + require.Equal(t, "test_master_log_file0000000000000000002", newBinlogPos) + require.Equal(t, true, ns.IsLoadingBinlog) +} + +// Написать такой же тест только наоборот +func TestUpdateBinlogWithoutChanges(t *testing.T) { + oldBinlogPostion := "test_master_log_file0000000000000000002" + ns := newMockNodeState() + + newBinlogPos := ns.UpdateBinlogStatus(oldBinlogPostion) + + require.Equal(t, "test_master_log_file0000000000000000002", newBinlogPos) + require.Equal(t, false, ns.IsLoadingBinlog) +} diff --git a/tests/features/binlog_loading.feature b/tests/features/binlog_loading.feature new file mode 100644 index 0000000..9a47bdd --- /dev/null +++ b/tests/features/binlog_loading.feature @@ -0,0 +1,31 @@ +Feature: loading binlogs tests + + Scenario: repl_mon enabled + Given cluster environment is + """ + REPL_MON=true + """ + Given cluster is up and running + When I wait for "10" seconds + Then zookeeper node "/test/health/mysql2" should match json within "20" seconds + """ + { + "is_loading_binlog": true + } + """ + + + Scenario: repl_mon disabled + Given cluster environment is + """ + REPL_MON=false + """ + Given cluster is up and running + When I wait for "10" seconds + Then zookeeper node "/test/health/mysql2" should match json within "20" seconds + """ + { + "is_loading_binlog": false + } + """ + From e0c454fe8118592d95aa202b6e684846563f2684 Mon Sep 17 00:00:00 2001 From: withsoull Date: Mon, 23 Dec 2024 14:52:40 +0300 Subject: [PATCH 03/11] Update Dockerfile: - removed 2 warnings --- tests/images/base/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/images/base/Dockerfile b/tests/images/base/Dockerfile index 98ab54b..377a233 100644 --- a/tests/images/base/Dockerfile +++ b/tests/images/base/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:jammy -ENV container docker -ENV DEBIAN_FRONTEND noninteractive +ENV container=docker +ENV DEBIAN_FRONTEND=noninteractive ENV ZK_VERSION=3.7.1 ARG MYSQL_VERSION="" ENV MYSQL_VERSION="${MYSQL_VERSION}" From e2c0cb1175b08225e76e11a3a46dc84c3c8fb0a2 Mon Sep 17 00:00:00 2001 From: withsoull Date: Mon, 23 Dec 2024 15:27:29 +0300 Subject: [PATCH 04/11] Add new func GetCurrentBinlogPosition() to SlaveState --- internal/app/app.go | 2 +- internal/app/data.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/app/app.go b/internal/app/app.go index abe2cb9..4d79024 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -965,7 +965,7 @@ func (app *App) calcActiveNodesChanges(clusterState map[string]*NodeState, activ slaveState := clusterState[host].SlaveState dataLag := calcLagBytes(masterBinlogs, slaveState.MasterLogFile, slaveState.MasterLogPos) if dataLag > app.config.SemiSyncEnableLag { - newBinLogPos := fmt.Sprintf("%s%019d", slaveState.MasterLogFile, slaveState.MasterLogPos) + newBinLogPos := slaveState.GetCurrentBinlogPosition() oldBinLogPos := app.slaveReadPositions[host] if newBinLogPos <= oldBinLogPos { diff --git a/internal/app/data.go b/internal/app/data.go index ccf7f5b..7fa114a 100644 --- a/internal/app/data.go +++ b/internal/app/data.go @@ -145,9 +145,13 @@ func (ns *NodeState) CalcGTIDDiffWithMaster() (string, error) { return gtids.GTIDDiff(replicaGTID, sourceGTID) } +func (ss *SlaveState) GetCurrentBinlogPosition() string { + return fmt.Sprintf("%s%019d", ss.MasterLogFile, ss.MasterLogPos) +} + func (ns *NodeState) UpdateBinlogStatus(oldBinloPos string) (newBinlogPos string) { if ns.SlaveState != nil { - newBinlogPos = fmt.Sprintf("%s%019d", ns.SlaveState.MasterLogFile, ns.SlaveState.MasterLogPos) + newBinlogPos = ns.SlaveState.GetCurrentBinlogPosition() if newBinlogPos > oldBinloPos { ns.IsLoadingBinlog = true From 08f85756308d38197dd42139dd85ccf4e6800360 Mon Sep 17 00:00:00 2001 From: withsoull Date: Mon, 23 Dec 2024 15:32:44 +0300 Subject: [PATCH 05/11] Some fixes during PR --- internal/app/data.go | 6 +----- internal/app/data_test.go | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/internal/app/data.go b/internal/app/data.go index 7fa114a..737197c 100644 --- a/internal/app/data.go +++ b/internal/app/data.go @@ -153,11 +153,7 @@ func (ns *NodeState) UpdateBinlogStatus(oldBinloPos string) (newBinlogPos string if ns.SlaveState != nil { newBinlogPos = ns.SlaveState.GetCurrentBinlogPosition() - if newBinlogPos > oldBinloPos { - ns.IsLoadingBinlog = true - } else { - ns.IsLoadingBinlog = false - } + ns.IsLoadingBinlog = newBinlogPos > oldBinloPos } return diff --git a/internal/app/data_test.go b/internal/app/data_test.go index 155611a..c6ff933 100644 --- a/internal/app/data_test.go +++ b/internal/app/data_test.go @@ -72,7 +72,6 @@ func TestUpdateBinlogWithChanges(t *testing.T) { require.Equal(t, true, ns.IsLoadingBinlog) } -// Написать такой же тест только наоборот func TestUpdateBinlogWithoutChanges(t *testing.T) { oldBinlogPostion := "test_master_log_file0000000000000000002" ns := newMockNodeState() From a0735374c1dd1566efaa5415e8dd891cba32941d Mon Sep 17 00:00:00 2001 From: withsoull Date: Tue, 24 Dec 2024 11:46:00 +0300 Subject: [PATCH 06/11] rename variable in data.go for correctly linting --- internal/app/data.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/app/data.go b/internal/app/data.go index 737197c..81678dd 100644 --- a/internal/app/data.go +++ b/internal/app/data.go @@ -264,15 +264,15 @@ type SlaveState struct { LastSQLErrno int `json:"last_sql_errno"` } -func (ns *SlaveState) FromReplicaStatus(replStatus mysql.ReplicaStatus) { - ns.ExecutedGtidSet = replStatus.GetExecutedGtidSet() - ns.RetrievedGtidSet = replStatus.GetRetrievedGtidSet() - ns.MasterHost = replStatus.GetMasterHost() - ns.ReplicationState = replStatus.ReplicationState() - ns.MasterLogFile = replStatus.GetMasterLogFile() - ns.MasterLogPos = replStatus.GetReadMasterLogPos() - ns.LastIOErrno = replStatus.GetLastIOErrno() - ns.LastSQLErrno = replStatus.GetLastSQLErrno() +func (ss *SlaveState) FromReplicaStatus(replStatus mysql.ReplicaStatus) { + ss.ExecutedGtidSet = replStatus.GetExecutedGtidSet() + ss.RetrievedGtidSet = replStatus.GetRetrievedGtidSet() + ss.MasterHost = replStatus.GetMasterHost() + ss.ReplicationState = replStatus.ReplicationState() + ss.MasterLogFile = replStatus.GetMasterLogFile() + ss.MasterLogPos = replStatus.GetReadMasterLogPos() + ss.LastIOErrno = replStatus.GetLastIOErrno() + ss.LastSQLErrno = replStatus.GetLastSQLErrno() } // SemiSyncState contains semi sync host settings From 754d4484201afe97ee34162b408acdd23f385a9e Mon Sep 17 00:00:00 2001 From: withsoull Date: Tue, 24 Dec 2024 12:52:55 +0300 Subject: [PATCH 07/11] Add testing binlog loading to repl_mon.feature testing --- tests/features/binlog_loading.feature | 31 ---------------------- tests/features/repl_mon.feature | 37 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 31 deletions(-) delete mode 100644 tests/features/binlog_loading.feature diff --git a/tests/features/binlog_loading.feature b/tests/features/binlog_loading.feature deleted file mode 100644 index 9a47bdd..0000000 --- a/tests/features/binlog_loading.feature +++ /dev/null @@ -1,31 +0,0 @@ -Feature: loading binlogs tests - - Scenario: repl_mon enabled - Given cluster environment is - """ - REPL_MON=true - """ - Given cluster is up and running - When I wait for "10" seconds - Then zookeeper node "/test/health/mysql2" should match json within "20" seconds - """ - { - "is_loading_binlog": true - } - """ - - - Scenario: repl_mon disabled - Given cluster environment is - """ - REPL_MON=false - """ - Given cluster is up and running - When I wait for "10" seconds - Then zookeeper node "/test/health/mysql2" should match json within "20" seconds - """ - { - "is_loading_binlog": false - } - """ - diff --git a/tests/features/repl_mon.feature b/tests/features/repl_mon.feature index bb94182..d3f6463 100644 --- a/tests/features/repl_mon.feature +++ b/tests/features/repl_mon.feature @@ -27,6 +27,25 @@ Feature: repl_mon tests [{"res":1}] """ + And mysql host "mysql2" should be replica of "mysql1" + + Then zookeeper node "/test/health/mysql2" should match json within "20" seconds + """ + { + "is_loading_binlog": true + } + """ + + And mysql host "mysql3" should be replica of "mysql1" + + Then zookeeper node "/test/health/mysql3" should match json within "20" seconds + """ + { + "is_loading_binlog": true + } + """ + + Scenario: repl_mon disabled Given cluster environment is """ @@ -51,3 +70,21 @@ Feature: repl_mon tests """ SELECT ts FROM mysql.mysync_repl_mon """ + + And mysql host "mysql2" should be replica of "mysql1" + + Then zookeeper node "/test/health/mysql2" should match json within "20" seconds + """ + { + "is_loading_binlog": false + } + """ + + And mysql host "mysql3" should be replica of "mysql1" + + Then zookeeper node "/test/health/mysql3" should match json within "20" seconds + """ + { + "is_loading_binlog": false + } + """ \ No newline at end of file From 0b319235da6c9039584a18a048744e0a9928cd94 Mon Sep 17 00:00:00 2001 From: withsoull Date: Tue, 24 Dec 2024 13:50:09 +0300 Subject: [PATCH 08/11] Microfix for linter --- tests/features/repl_mon.feature | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/features/repl_mon.feature b/tests/features/repl_mon.feature index d3f6463..da68ca9 100644 --- a/tests/features/repl_mon.feature +++ b/tests/features/repl_mon.feature @@ -87,4 +87,5 @@ Feature: repl_mon tests { "is_loading_binlog": false } - """ \ No newline at end of file + """ + \ No newline at end of file From 87fdb24bfad78fa517cf728b2f141ec658ffd2ac Mon Sep 17 00:00:00 2001 From: withsoull Date: Tue, 24 Dec 2024 15:31:02 +0300 Subject: [PATCH 09/11] microfixes for linter --- tests/features/repl_mon.feature | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/features/repl_mon.feature b/tests/features/repl_mon.feature index da68ca9..edb2b97 100644 --- a/tests/features/repl_mon.feature +++ b/tests/features/repl_mon.feature @@ -88,4 +88,3 @@ Feature: repl_mon tests "is_loading_binlog": false } """ - \ No newline at end of file From 6ad8809b99101c6c22d908bb98d3ff3a39724a10 Mon Sep 17 00:00:00 2001 From: withsoull Date: Tue, 24 Dec 2024 15:40:39 +0300 Subject: [PATCH 10/11] Remove gaps --- tests/features/repl_mon.feature | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/features/repl_mon.feature b/tests/features/repl_mon.feature index edb2b97..16fe004 100644 --- a/tests/features/repl_mon.feature +++ b/tests/features/repl_mon.feature @@ -70,18 +70,14 @@ Feature: repl_mon tests """ SELECT ts FROM mysql.mysync_repl_mon """ - And mysql host "mysql2" should be replica of "mysql1" - Then zookeeper node "/test/health/mysql2" should match json within "20" seconds """ { "is_loading_binlog": false } """ - And mysql host "mysql3" should be replica of "mysql1" - Then zookeeper node "/test/health/mysql3" should match json within "20" seconds """ { From 315663f2cf998bc29026f28a4976b916d1ef04ad Mon Sep 17 00:00:00 2001 From: withsoull Date: Tue, 24 Dec 2024 16:02:46 +0300 Subject: [PATCH 11/11] Remove another gaps --- tests/features/repl_mon.feature | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/features/repl_mon.feature b/tests/features/repl_mon.feature index 16fe004..95dd951 100644 --- a/tests/features/repl_mon.feature +++ b/tests/features/repl_mon.feature @@ -26,18 +26,14 @@ Feature: repl_mon tests """ [{"res":1}] """ - And mysql host "mysql2" should be replica of "mysql1" - Then zookeeper node "/test/health/mysql2" should match json within "20" seconds """ { "is_loading_binlog": true } """ - And mysql host "mysql3" should be replica of "mysql1" - Then zookeeper node "/test/health/mysql3" should match json within "20" seconds """ { @@ -45,7 +41,6 @@ Feature: repl_mon tests } """ - Scenario: repl_mon disabled Given cluster environment is """