Merge pull request #360 from fernandoroyosanchez/ovn_controller_healt…

…hchecks

Add Liveness and Readiness probes to ovn-controller

pkg/ovncontroller/daemonset.go

@@ -62,6 +62,32 @@ func CreateOVNDaemonSet(
 		}...)
 	}
 
+	ovnControllerLivenessProbe := &corev1.Probe{
+		// TODO might need tuning
+		TimeoutSeconds:      5,
+		PeriodSeconds:       5,
+		InitialDelaySeconds: 30,
+	}
+
+	ovnControllerReadinessProbe := &corev1.Probe{
+		// TODO might need tuning
+		TimeoutSeconds:      5,
+		PeriodSeconds:       5,
+		InitialDelaySeconds: 30,
+	}
+
+	ovnControllerLivenessProbe.Exec = &corev1.ExecAction{
+		Command: []string{
+			"/usr/local/bin/container-scripts/ovn_controller_liveness.sh",
+		},
+	}
+
+	ovnControllerReadinessProbe.Exec = &corev1.ExecAction{
+		Command: []string{
+			"/usr/local/bin/container-scripts/ovn_controller_readiness.sh",
+		},
+	}
+
 	runAsUser := int64(0)
 	privileged := true
 
@@ -88,8 +114,10 @@ func CreateOVNDaemonSet(
 				RunAsUser:  &runAsUser,
 				Privileged: &privileged,
 			},
-			Env:          env.MergeEnvs([]corev1.EnvVar{}, envVars),
-			VolumeMounts: mounts,
+			Env:            env.MergeEnvs([]corev1.EnvVar{}, envVars),
+			VolumeMounts:   mounts,
+			ReadinessProbe: ovnControllerReadinessProbe,
+			LivenessProbe:  ovnControllerLivenessProbe,
 			// TODO: consider the fact that resources are now double booked
 			Resources:                instance.Spec.Resources,
 			TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
@@ -148,16 +176,39 @@ func CreateOVSDaemonSet(
 		InitialDelaySeconds: 3,
 	}
 
+	ovsDbReadinessProbe := &corev1.Probe{
+		// TODO might need tuning
+		TimeoutSeconds:      5,
+		PeriodSeconds:       3,
+		InitialDelaySeconds: 3,
+	}
+
+	ovsVswitchdReadinessProbe := &corev1.Probe{
+		// TODO might need tuning
+		TimeoutSeconds:      5,
+		PeriodSeconds:       3,
+		InitialDelaySeconds: 3,
+	}
+
 	ovsDbLivenessProbe.Exec = &corev1.ExecAction{
 		Command: []string{
-			"/usr/bin/ovs-vsctl",
-			"show",
+			"/usr/local/bin/container-scripts/ovsdb_server_liveness.sh",
 		},
 	}
 	ovsVswitchdLivenessProbe.Exec = &corev1.ExecAction{
 		Command: []string{
-			"/usr/bin/ovs-appctl",
-			"bond/show",
+			"/usr/local/bin/container-scripts/vswitchd_liveness.sh",
+		},
+	}
+
+	ovsDbReadinessProbe.Exec = &corev1.ExecAction{
+		Command: []string{
+			"/usr/local/bin/container-scripts/ovsdb_server_readiness.sh",
+		},
+	}
+	ovsVswitchdReadinessProbe.Exec = &corev1.ExecAction{
+		Command: []string{
+			"/usr/local/bin/container-scripts/vswitchd_readiness.sh",
 		},
 	}
 
@@ -211,6 +262,7 @@ func CreateOVSDaemonSet(
 			// TODO: consider the fact that resources are now double booked
 			Resources:                instance.Spec.Resources,
 			LivenessProbe:            ovsDbLivenessProbe,
+			ReadinessProbe:           ovsDbReadinessProbe,
 			TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
 		},
 		{
@@ -237,6 +289,7 @@ func CreateOVSDaemonSet(
 			// TODO: consider the fact that resources are now double booked
 			Resources:                instance.Spec.Resources,
 			LivenessProbe:            ovsVswitchdLivenessProbe,
+			ReadinessProbe:           ovsVswitchdReadinessProbe,
 			TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
 		},
 	}

templates/ovncontroller/bin/ovn_controller_liveness.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+error_exit() {
+    echo "$1" >&2
+    exit 1
+}
+
+# Check if ovn-controller is running
+check_ovn_controller_pid() {
+    if ! pidof -q ovn-controller; then
+        error_exit "ERROR - ovn-controller is not running"
+    fi
+}
+
+# Function to check the running status of ovn-controller
+check_ovn_controller_status() {
+    # Capture output and check exit status
+    if ! output=$(ovn-appctl -t ovn-controller debug/status 2>&1); then
+        error_exit "ERROR - Failed to get status from ovn-controller, ovn-appctl exit status: $?"
+    fi
+
+    if [ "$output" != "running" ]; then
+        error_exit "ERROR - ovn-controller status is '$output', expecting 'running' status"
+    fi
+}
+
+
+check_ovn_controller_pid
+check_ovn_controller_status

templates/ovncontroller/bin/ovn_controller_readiness.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+error_exit() {
+    echo "$1" >&2
+    exit 1
+}
+
+# Check if ovn-controller is connected to the OVN SB database
+check_ovn_controller_connection() {
+    if ! output=$(ovn-appctl -t ovn-controller connection-status 2>&1); then
+        error_exit "ERROR - Failed to get connection status from ovn-controller, ovn-appctl exit status: $?"
+    fi
+
+    if [ "$output" != "connected" ]; then
+        error_exit "ERROR - ovn-controller connection status is '$output', expecting 'connected' status"
+    fi
+}
+
+
+check_ovn_controller_connection

templates/ovncontroller/bin/ovsdb_server_liveness.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+error_exit() {
+    echo "$1" >&2
+    exit 1
+}
+
+# Check if ovsdb-server is running
+check_ovsdb_server_pid() {
+    if ! pidof -q ovsdb-server; then
+        error_exit "ERROR - ovsdb-server is not running"
+    fi
+}
+
+# Function to check the running status of ovsdb-server
+check_ovsdb_server_status() {
+    if ! /usr/bin/ovs-vsctl show 2>&1; then
+        error_exit "ERROR - Failed to get output from ovsdb-server, ovs-vsctl exit status: $?"
+    fi
+}
+
+
+check_ovsdb_server_pid
+check_ovsdb_server_status

templates/ovncontroller/bin/ovsdb_server_readiness.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+error_exit() {
+    echo "$1" >&2
+    exit 1
+}
+
+# Check ovsdb-server status
+check_ovsdb_server_status() {
+
+    if ! pid=$(cat /var/run/openvswitch/ovsdb-server.pid); then
+        error_exit "ERROR - Failed to get pid for ovsdb-server, exit status: $?"
+    fi
+
+    if ! output=$(ovs-appctl -t /var/run/openvswitch/ovsdb-server."$pid".ctl ovsdb-server/list-dbs); then
+        error_exit "ERROR - Failed retrieving list of databases from ovsdb-server"
+    fi
+
+    if [[ "$output" != *"Open_vSwitch"* ]]; then
+        error_exit "ERROR - 'Open_vSwitch' not found on databases"
+    fi
+}
+
+check_ovsdb_server_status

templates/ovncontroller/bin/vswitchd_liveness.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+error_exit() {
+    echo "$1" >&2
+    exit 1
+}
+
+# Check if ovs-vswitchd is running
+check_ovs_vswitchd_pid() {
+    if ! pidof -q ovs-vswitchd; then
+        error_exit "ERROR - ovs-vswitchd is not running"
+    fi
+}
+
+# Function to check the running status of ovs-vswitchd
+check_ovs_vswitchd_status() {
+    if ! /usr/bin/ovs-appctl bond/show 2>&1; then
+        error_exit "ERROR - Failed to get output from ovs-vswitchd, ovs-appctl exit status: $?"
+    fi
+}
+
+
+check_ovs_vswitchd_pid
+check_ovs_vswitchd_status

templates/ovncontroller/bin/vswitchd_readiness.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+error_exit() {
+    echo "$1" >&2
+    exit 1
+}
+
+# Check ovs-vswitchd status
+check_ovs_vswitchd_status() {
+
+    if ! pid=$(cat /var/run/openvswitch/ovs-vswitchd.pid); then
+        error_exit "ERROR - Failed to get pid for ovs-vswitchd, exit status: $?"
+    fi
+
+    if ! output=$(ovs-appctl -t /var/run/openvswitch/ovs-vswitchd."$pid".ctl ofproto/list); then
+        error_exit "ERROR - Failed retrieving ofproto/list from ovs-vswitchd"
+    fi
+
+    if [ -z "$output" ]; then
+        error_exit "ERROR - No bridges on ofproto/list output"
+    fi
+}
+
+check_ovs_vswitchd_status