Merge branch 'master' into jepsen-ssh-fix

README.md

@@ -116,6 +116,7 @@ replication_repair_max_attempts: 3
 
 external_replication_type: off
 show_only_gtid_diff: False
+force_switchover: False
 ```
 
 ### Usage

internal/app/app.go

@@ -1208,6 +1208,21 @@ func (app *App) performSwitchover(clusterState map[string]*NodeState, activeNode
 		}
 		node := app.cluster.Get(host)
 		// in case node is a master
+
+		if app.config.ForceSwitchover {
+			err := node.SetOfflineForce()
+			if err != nil {
+				return fmt.Errorf("failed to set node %s force offline: %v", host, err)
+			}
+
+			defer func() {
+				err := node.SetOnline()
+				if err != nil {
+					app.logger.Errorf("failed to set node %s online after setting force offline: %v", host, err)
+				}
+			}()
+		}
+
 		err := node.SetReadOnly(true)
 		if err != nil || app.emulateError("freeze_ro") {
 			app.logger.Infof("switchover: failed to set node %s read-only, trying kill bad queries: %v", host, err)

internal/config/config.go

@@ -97,6 +97,7 @@ type Config struct {
 	ReplMonErrorWaitInterval                time.Duration                `config:"repl_mon_error_wait_interval" yaml:"repl_mon_error_wait_interval"`
 	ReplMonSlaveWaitInterval                time.Duration                `config:"repl_mon_slave_wait_interval" yaml:"repl_mon_slave_wait_interval"`
 	ShowOnlyGTIDDiff                        bool                         `config:"show_only_gtid_diff" yaml:"show_only_gtid_diff"`
+	ForceSwitchover                         bool                         `config:"force_switchover" yaml:"force_switchover"` // TODO: Remove when we will be sure it's right way to do switchover
 }
 
 // DefaultConfig returns default configuration for MySync
@@ -182,6 +183,7 @@ func DefaultConfig() (Config, error) {
 		ReplMonErrorWaitInterval:                10 * time.Second,
 		ReplMonSlaveWaitInterval:                10 * time.Second,
 		ShowOnlyGTIDDiff:                        false,
+		ForceSwitchover:                         false,
 	}
 	return config, nil
 }

internal/mysql/node.go

@@ -833,6 +833,14 @@ func (n *Node) SetOnline() error {
 	return n.exec(queryDisableOfflineMode, nil)
 }
 
+func (n *Node) SetOfflineForce() error {
+	err := n.SemiSyncDisable()
+	if err != nil {
+		return err
+	}
+	return n.SetOffline()
+}
+
 // ChangeMaster changes master of MySQL Node, demoting it to slave
 func (n *Node) ChangeMaster(host string) error {
 	useSsl := 0

tests/features/switchover_from.feature

@@ -37,7 +37,11 @@ Feature: manual switchover from old master
       }
       """
 
-  Scenario: if switchover was approved, it will not be rejected
+  Scenario Outline: if switchover was approved, it will not be rejected
+    Given cluster environment is
+      """
+      FORCE_SWITCHOVER=<force_switchover>
+      """
     Given cluster is up and running
     Then zookeeper node "/test/active_nodes" should match json_exactly within "20" seconds
       """
@@ -71,6 +75,10 @@ Feature: manual switchover from old master
           }
       }
       """
+    Examples:
+      | force_switchover  |
+      | true              |
+      | false             |
 
   Scenario Outline: switchover from works on healthy cluster
     Given cluster environment is

tests/features/switchover_to.feature

@@ -1,7 +1,11 @@
 Feature: manual switchover to new master
 
 
-  Scenario: switchover on kill all running query on old master
+  Scenario Outline: switchover on kill all running query on old master
+    Given cluster environment is
+    """
+    FORCE_SWITCHOVER=<force_switchover>
+    """
     Given cluster is up and running
     Then mysql host "mysql1" should be master
     And mysql host "mysql2" should be replica of "mysql1"
@@ -54,6 +58,10 @@ Feature: manual switchover to new master
     And mysql host "mysql3" should have variable "rpl_semi_sync_master_enabled" set to "0"
     And mysql replication on host "mysql3" should run fine within "3" seconds
     And mysql host "mysql3" should be read only
+    Examples:
+      | force_switchover  |
+      | true              |
+      | false             |
 
   Scenario Outline: switchover to works on healthy cluster
     Given cluster environment is
@@ -274,4 +282,3 @@ Feature: manual switchover to new master
     """
     mysql2 is not active
     """
-

tests/images/mysql/mysync.yaml

@@ -62,3 +62,4 @@ replication_channel: ''
 external_replication_type: 'external'
 show_only_gtid_diff: false
 repl_mon: ${REPL_MON:-false}
+force_switchover: ${FORCE_SWITCHOVER:-false}