Improved Hub XML-RPC support

mgradm/cmd/inspect/kubernetes.go

@@ -13,7 +13,6 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 
-	adm_utils "github.com/uyuni-project/uyuni-tools/mgradm/shared/utils"
 	"github.com/uyuni-project/uyuni-tools/shared"
 	shared_kubernetes "github.com/uyuni-project/uyuni-tools/shared/kubernetes"
 	. "github.com/uyuni-project/uyuni-tools/shared/l10n"
@@ -36,7 +35,7 @@ func kuberneteInspect(
 	if len(serverImage) <= 0 {
 		log.Debug().Msg("Use deployed image")
 
-		serverImage, err = adm_utils.RunningImage(cnx)
+		serverImage, err = shared_kubernetes.GetRunningImage("uyuni")
 		if err != nil {
 			return errors.New(L("failed to find the image of the currently running server container: %s"))
 		}

mgradm/cmd/inspect/podman.go

@@ -9,10 +9,8 @@ import (
 
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
-	adm_utils "github.com/uyuni-project/uyuni-tools/mgradm/shared/utils"
-	"github.com/uyuni-project/uyuni-tools/shared"
 	. "github.com/uyuni-project/uyuni-tools/shared/l10n"
-	shared_podman "github.com/uyuni-project/uyuni-tools/shared/podman"
+	"github.com/uyuni-project/uyuni-tools/shared/podman"
 	"github.com/uyuni-project/uyuni-tools/shared/types"
 	"github.com/uyuni-project/uyuni-tools/shared/utils"
 )
@@ -31,13 +29,12 @@ func podmanInspect(
 	if len(serverImage) <= 0 {
 		log.Debug().Msg("Use deployed image")
 
-		cnx := shared.NewConnection("podman", shared_podman.ServerContainerName, "")
-		serverImage, err = adm_utils.RunningImage(cnx)
+		serverImage, err = podman.GetRunningImage(podman.ServerContainerName)
 		if err != nil {
 			return utils.Errorf(err, L("failed to find the image of the currently running server container"))
 		}
 	}
-	inspectResult, err := shared_podman.Inspect(serverImage, flags.Image.PullPolicy, flags.SCC)
+	inspectResult, err := podman.Inspect(serverImage, flags.Image.PullPolicy, flags.SCC)
 	if err != nil {
 		return utils.Errorf(err, L("inspect command failed"))
 	}

mgradm/cmd/install/kubernetes/utils.go

@@ -74,7 +74,8 @@ func installForKubernetes(_ *types.GlobalFlags,
 	}
 
 	// Deploy Uyuni and wait for it to be up
-	if err := kubernetes.Deploy(cnx, flags.Image.Registry, &flags.Image, &flags.Helm,
+	if err := kubernetes.Deploy(
+		cnx, flags.Image.Registry, &flags.Image, &flags.HubXmlrpc, &flags.Helm,
 		clusterInfos, fqdn, flags.Debug.Java, false, helmArgs...,
 	); err != nil {
 		return shared_utils.Errorf(err, L("cannot deploy uyuni"))

mgradm/cmd/migrate/kubernetes/utils.go

@@ -48,6 +48,12 @@ func migrateToKubernetes(
 		return utils.Errorf(err, L("failed to compute image URL"))
 	}
 
+	hubXmlrpcImage := ""
+	hubXmlrpcImage, err = utils.ComputeImage(flags.Image.Registry, flags.Image.Tag, flags.HubXmlrpc.Image)
+	if err != nil {
+		return err
+	}
+
 	fqdn := args[0]
 	if err := utils.IsValidFQDN(fqdn); err != nil {
 		return err
@@ -89,8 +95,9 @@ func migrateToKubernetes(
 		"--set", "migration.dataPath="+scriptDir,
 	)
 
-	if err := kubernetes.Deploy(cnx, flags.Image.Registry, &flags.Image, &flags.Helm,
-		clusterInfos, fqdn, false, flags.Prepare, migrationArgs...); err != nil {
+	if err := kubernetes.Deploy(cnx, flags.Image.Registry, &flags.Image, &flags.HubXmlrpc,
+		&flags.Helm, clusterInfos, fqdn, false, flags.Prepare, migrationArgs...,
+	); err != nil {
 		return utils.Errorf(err, L("cannot run deploy"))
 	}
 
@@ -145,8 +152,10 @@ func migrateToKubernetes(
 	helmArgs = append(helmArgs, setupSSLArray...)
 
 	// Run uyuni upgrade using the new ssl certificate
+	// We don't need to start the Hub XML-RPC API containers during the setup phase
 	if err = kubernetes.UyuniUpgrade(
-		serverImage, flags.Image.PullPolicy, &flags.Helm, kubeconfig, fqdn, clusterInfos.Ingress, helmArgs...,
+		serverImage, flags.Image.PullPolicy, 0, hubXmlrpcImage, &flags.Helm,
+		kubeconfig, fqdn, clusterInfos.Ingress, helmArgs...,
 	); err != nil {
 		return utils.Errorf(err, L("cannot upgrade helm chart to image %s using new SSL certificate"), serverImage)
 	}
@@ -182,8 +191,16 @@ func migrateToKubernetes(
 		return utils.Errorf(err, L("cannot run post upgrade script"))
 	}
 
+	hubReplicas := flags.HubXmlrpc.Replicas
+	if extractedData.HasHubXmlrpcAPI {
+		log.Info().Msg(L("Enabling Hub XML-RPC API since it is enabled on the migrated server"))
+		hubReplicas = 1
+	}
+
+	// This is the final deployment, all the replicas need to be correct here.
 	if err := kubernetes.UyuniUpgrade(
-		serverImage, flags.Image.PullPolicy, &flags.Helm, kubeconfig, fqdn, clusterInfos.Ingress, helmArgs...,
+		serverImage, flags.Image.PullPolicy, hubReplicas, hubXmlrpcImage, &flags.Helm, kubeconfig,
+		fqdn, clusterInfos.Ingress, helmArgs...,
 	); err != nil {
 		return utils.Errorf(err, L("cannot upgrade to image %s"), serverImage)
 	}
@@ -192,6 +209,8 @@ func migrateToKubernetes(
 		return err
 	}
 
+	// Reinitialize the connection since the pod name has changed since we first checked
+	cnx = shared.NewConnection("kubectl", "", shared_kubernetes.ServerFilter)
 	if err := cnx.CopyCaCertificate(fqdn); err != nil {
 		return utils.Errorf(err, L("failed to add SSL CA certificate to host trusted certificates"))
 	}

mgradm/cmd/scale/podman.go

@@ -31,7 +31,7 @@ func podmanScale(
 	}
 	if service == podman.HubXmlrpcService {
 		if newReplicas > 1 {
-			return errors.New(L("Multiple Hub XML-RPC container replicas are not currently supported."))
+			return errors.New(L("Multiple Hub XML-RPC API container replicas are not currently supported."))
 		}
 		return systemd.ScaleService(newReplicas, service)
 	}

mgradm/cmd/upgrade/kubernetes/utils.go

@@ -18,5 +18,7 @@ func upgradeKubernetes(
 	cmd *cobra.Command,
 	args []string,
 ) error {
-	return kubernetes.Upgrade(globalFlags, &flags.Image, &flags.DBUpgradeImage, flags.Helm, cmd, args)
+	return kubernetes.Upgrade(
+		globalFlags, &flags.UpgradeFlags.Image, &flags.DBUpgradeImage, &flags.HubXmlrpc.Image, flags.Helm, cmd, args,
+	)
 }

mgradm/shared/kubernetes/certificates.go

@@ -113,6 +113,7 @@ func installCertManager(helmFlags *cmd_utils.HelmFlags, kubeconfig string, image
 
 		args := []string{
 			"--set", "crds.enabled=true",
+			"--set", "crds.keep=true",
 			"--set-json", "global.commonLabels={\"installedby\": \"mgradm\"}",
 			"--set", "image.pullPolicy=" + kubernetes.GetPullPolicy(imagePullPolicy),
 		}

mgradm/shared/kubernetes/install.go

@@ -28,6 +28,7 @@ func Deploy(
 	cnx *shared.Connection,
 	registry string,
 	imageFlags *types.ImageFlags,
+	hubXmlrpcFlags *cmd_utils.HubXmlrpcFlags,
 	helmFlags *cmd_utils.HelmFlags,
 	clusterInfos *kubernetes.ClusterInfos,
 	fqdn string,
@@ -39,10 +40,11 @@ func Deploy(
 	isK3s := clusterInfos.IsK3s()
 	IsRke2 := clusterInfos.IsRke2()
 	if !prepare {
+		tcpPorts, udpPorts := GetPortLists(hubXmlrpcFlags.Replicas > 0, debug)
 		if isK3s {
-			InstallK3sTraefikConfig(debug)
+			kubernetes.InstallK3sTraefikConfig(tcpPorts, udpPorts)
 		} else if IsRke2 {
-			kubernetes.InstallRke2NginxConfig(utils.TCPPorts, utils.UDPPorts, helmFlags.Uyuni.Namespace)
+			kubernetes.InstallRke2NginxConfig(tcpPorts, udpPorts, helmFlags.Uyuni.Namespace)
 		}
 	}
 
@@ -51,9 +53,15 @@ func Deploy(
 		return utils.Errorf(err, L("failed to compute image URL"))
 	}
 
+	hubXmlrpcImage, err := utils.ComputeImage(registry, imageFlags.Tag, hubXmlrpcFlags.Image)
+	if err != nil {
+		return utils.Errorf(err, L("failed to compute image URL"))
+	}
+
 	// Install the uyuni server helm chart
 	if err := UyuniUpgrade(
-		serverImage, imageFlags.PullPolicy, helmFlags, clusterInfos.GetKubeconfig(), fqdn, clusterInfos.Ingress, helmArgs...,
+		serverImage, imageFlags.PullPolicy, hubXmlrpcFlags.Replicas, hubXmlrpcImage, helmFlags,
+		clusterInfos.GetKubeconfig(), fqdn, clusterInfos.Ingress, helmArgs...,
 	); err != nil {
 		return utils.Errorf(err, L("cannot upgrade"))
 	}
@@ -66,7 +74,7 @@ func Deploy(
 	return cnx.WaitForServer()
 }
 
-// DeployCertificate executre a deploy a new certificate given an helm.
+// DeployCertificate deploys a new SSL certificate.
 func DeployCertificate(helmFlags *cmd_utils.HelmFlags, sslFlags *cmd_utils.InstallSSLFlags, rootCa string,
 	ca *types.SSLPair, kubeconfig string, fqdn string, imagePullPolicy string) ([]string, error) {
 	helmArgs := []string{}
@@ -107,8 +115,17 @@ func DeployExistingCertificate(
 }
 
 // UyuniUpgrade runs an helm upgrade using images and helm configuration as parameters.
-func UyuniUpgrade(serverImage string, pullPolicy string, helmFlags *cmd_utils.HelmFlags, kubeconfig string,
-	fqdn string, ingress string, helmArgs ...string) error {
+func UyuniUpgrade(
+	serverImage string,
+	pullPolicy string,
+	hubXmlrpcReplicas int,
+	hubXmlrpcImage string,
+	helmFlags *cmd_utils.HelmFlags,
+	kubeconfig string,
+	fqdn string,
+	ingress string,
+	helmArgs ...string,
+) error {
 	log.Info().Msg(L("Installing Uyuni"))
 
 	// The guessed ingress is passed before the user's value to let the user override it in case we got it wrong.
@@ -127,6 +144,12 @@ func UyuniUpgrade(serverImage string, pullPolicy string, helmFlags *cmd_utils.He
 		"--set", "pullPolicy="+kubernetes.GetPullPolicy(pullPolicy),
 		"--set", "fqdn="+fqdn)
 
+	if hubXmlrpcReplicas > 0 {
+		log.Info().Msg(L("Enabling Hub XMLRPC API container."))
+		helmParams = append(helmParams,
+			"--set", fmt.Sprintf("hub.api.replicas=%v", hubXmlrpcReplicas),
+			"--set", "images.hub_xmlrpc="+hubXmlrpcImage)
+	}
 	helmParams = append(helmParams, helmArgs...)
 
 	namespace := helmFlags.Uyuni.Namespace
@@ -140,6 +163,7 @@ func Upgrade(
 	_ *types.GlobalFlags,
 	image *types.ImageFlags,
 	upgradeImage *types.ImageFlags,
+	hubXmlrpcImage *types.ImageFlags,
 	helm cmd_utils.HelmFlags,
 	_ *cobra.Command,
 	_ []string,
@@ -156,6 +180,11 @@ func Upgrade(
 		return utils.Errorf(err, L("failed retrieving namespace"))
 	}
 
+	origHubXmlrpcImage, err := kubernetes.GetRunningImage("hub-xmlrpc-api")
+	if err != nil {
+		return utils.Errorf(err, L("failed to find Hub XML-RPC API container"))
+	}
+
 	serverImage, err := utils.ComputeImage(image.Registry, utils.DefaultTag, *image)
 	if err != nil {
 		return utils.Errorf(err, L("failed to compute image URL"))
@@ -238,7 +267,20 @@ func Upgrade(
 		helmArgs = append(helmArgs, "--set", "registrySecret="+pullSecret)
 	}
 
-	err = UyuniUpgrade(serverImage, image.PullPolicy, &helm, kubeconfig, fqdn, clusterInfos.Ingress, helmArgs...)
+	hubXmlrpcImageName, err := utils.ComputeImage(image.Registry, image.Tag, *hubXmlrpcImage)
+	if err != nil {
+		return utils.Errorf(err, L("failed to compute image URL"))
+	}
+
+	hubXmlrpcReplicas := 0
+	if origHubXmlrpcImage != "" {
+		hubXmlrpcReplicas = 1
+	}
+
+	err = UyuniUpgrade(
+		serverImage, image.PullPolicy, hubXmlrpcReplicas, hubXmlrpcImageName, &helm, kubeconfig, fqdn,
+		clusterInfos.Ingress, helmArgs...,
+	)
 	if err != nil {
 		return utils.Errorf(err, L("cannot upgrade to image %s"), serverImage)
 	}

mgradm/shared/kubernetes/k3s.go

@@ -15,15 +15,18 @@ import (
 	"github.com/uyuni-project/uyuni-tools/shared/utils"
 )
 
-// InstallK3sTraefikConfig installs the K3s Traefik configuration.
-func InstallK3sTraefikConfig(debug bool) {
+// GetPortLists returns compiled lists of tcp and udp ports..
+func GetPortLists(hub bool, debug bool) ([]types.PortMap, []types.PortMap) {
 	tcpPorts := []types.PortMap{}
 	tcpPorts = append(tcpPorts, utils.TCPPorts...)
 	if debug {
 		tcpPorts = append(tcpPorts, utils.DebugPorts...)
 	}
+	if hub {
+		tcpPorts = append(tcpPorts, utils.HubXmlrpcPorts...)
+	}
 
-	kubernetes.InstallK3sTraefikConfig(tcpPorts, utils.UDPPorts)
+	return tcpPorts, utils.UDPPorts
 }
 
 // RunPgsqlVersionUpgrade perform a PostgreSQL major upgrade.

mgradm/shared/templates/migrateScriptTemplate.go

@@ -126,8 +126,10 @@ echo "Extracting time zone..."
 $SSH {{ .SourceFqdn }} timedatectl show -p Timezone >/var/lib/uyuni-tools/data
 
 echo "Extracting postgresql versions..."
-echo "image_pg_version=$(rpm -qa --qf '%{VERSION}\n' 'name=postgresql[0-8][0-9]-server'  | cut -d. -f1 | sort -n | tail -1)" >> /var/lib/uyuni-tools/data
-echo "current_pg_version=$(cat /var/lib/pgsql/data/PG_VERSION)" >> /var/lib/uyuni-tools/data
+image_pg_version=$(rpm -qa --qf '%{VERSION}\n' 'name=postgresql[0-8][0-9]-server'  | cut -d. -f1 | sort -n | tail -1)
+current_pg_version=$(cat /var/lib/pgsql/data/PG_VERSION)
+echo "image_pg_version=$image_pg_version" >> /var/lib/uyuni-tools/data
+echo "current_pg_version=$current_pg_version" >> /var/lib/uyuni-tools/data
 
 grep '^db_user' /etc/rhn/rhn.conf | sed 's/[ \t]//g' >>/var/lib/uyuni-tools/data
 grep '^db_password' /etc/rhn/rhn.conf | sed 's/[ \t]//g' >>/var/lib/uyuni-tools/data
@@ -151,6 +153,15 @@ sed 's/--add-modules java.annotation,com.sun.xml.bind://' -i /etc/tomcat/conf.d/
 sed 's/-XX:-UseConcMarkSweepGC//' -i /etc/tomcat/conf.d/*
 test -f /etc/tomcat/conf.d/remote_debug.conf && sed 's/address=[^:]*:/address=*:/' -i /etc/tomcat/conf.d/remote_debug.conf
 
+# Create a backup copy of the data to prepare DB upgrade.
+# We need to upgrade the deployment before upgrading the database to get the SSL certificates ready.
+# To avoid corrupting the database files, move them to where the upgrade script will expect them.
+echo "Posgresql versions: image: $image_pg_version, current: $current_pg_version"
+if test "$image_pg_version" != "$current_pg_version"; then
+    echo "Backing up the database files ..."
+	mv /var/lib/pgsql/data /var/lib/pgsql/data-pg$current_pg_version
+fi
+
 {{ if .Kubernetes }}
 echo 'server.no_ssl = 1' >> /etc/rhn/rhn.conf;
 echo "Extracting SSL certificate and authority"

mgradm/shared/templates/pgsqlVersionUpgradeScriptTemplate.go

@@ -23,8 +23,9 @@ test -d /usr/lib/postgresql$NEW_VERSION/bin
 echo "Testing presence of postgresql$OLD_VERSION..."
 test -d /usr/lib/postgresql$OLD_VERSION/bin
 
-echo "Create a backup at /var/lib/pgsql/data-pg$OLD_VERSION..."
-mv /var/lib/pgsql/data /var/lib/pgsql/data-pg$OLD_VERSION
+# Data have already been backed up at the end of the migration script
+# Reset the potentially created new pgsql data
+rm -rf /var/lib/pgsql/data
 echo "Create new database directory..."
 mkdir -p /var/lib/pgsql/data
 chown -R postgres:postgres /var/lib/pgsql
@@ -45,6 +46,8 @@ echo "Running initdb using postgres user"
 echo "Any suggested command from the console should be run using postgres user"
 su -s /bin/bash - postgres -c "initdb -D /var/lib/pgsql/data --locale=$POSTGRES_LANG"
 echo "Successfully initialized new postgresql $NEW_VERSION database."
+
+
 su -s /bin/bash - postgres -c "pg_upgrade --old-bindir=/usr/lib/postgresql$OLD_VERSION/bin --new-bindir=/usr/lib/postgresql$NEW_VERSION/bin --old-datadir=/var/lib/pgsql/data-pg$OLD_VERSION --new-datadir=/var/lib/pgsql/data $FAST_UPGRADE"
 
 cp /var/lib/pgsql/data-pg$OLD_VERSION/pg_hba.conf /var/lib/pgsql/data

mgradm/shared/utils/exec.go

@@ -15,7 +15,6 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/uyuni-project/uyuni-tools/mgradm/shared/templates"
 	"github.com/uyuni-project/uyuni-tools/shared"
-	"github.com/uyuni-project/uyuni-tools/shared/kubernetes"
 	. "github.com/uyuni-project/uyuni-tools/shared/l10n"
 	"github.com/uyuni-project/uyuni-tools/shared/utils"
 )
@@ -137,37 +136,6 @@ func GenerateMigrationScript(sourceFqdn string, user string, kubernetes bool, pr
 	return scriptDir, cleaner, nil
 }
 
-// RunningImage returns the image running in the current system.
-func RunningImage(cnx *shared.Connection) (string, error) {
-	command, err := cnx.GetCommand()
-
-	switch command {
-	case "podman":
-		args := []string{"ps", "--format", "{{.Image}}", "--noheading"}
-		image, err := utils.RunCmdOutput(zerolog.DebugLevel, "podman", args...)
-		if err != nil {
-			return "", err
-		}
-		return strings.Trim(string(image), "\n"), nil
-
-	case "kubectl":
-
-		// FIXME this will work until containers 0 is uyuni. Then jsonpath should be something like
-		// {.items[0].spec.containers[?(@.name=="` + containerName + `")].image but there are problems
-		// using RunCmdOutput with an arguments with round brackets
-		args := []string{"get", "pods", kubernetes.ServerFilter, "-o", "jsonpath={.items[0].spec.containers[0].image}"}
-		image, err := utils.RunCmdOutput(zerolog.DebugLevel, "kubectl", args...)
-
-		log.Info().Msgf(L("Image is: %s"), image)
-		if err != nil {
-			return "", err
-		}
-		return strings.Trim(string(image), "\n"), nil
-	}
-
-	return command, err
-}
-
 // SanityCheck verifies if an upgrade can be run.
 func SanityCheck(cnx *shared.Connection, inspectedValues *utils.ServerInspectData, serverImage string) error {
 	isUyuni, err := isUyuni(cnx)

shared/kubernetes/k3s.go

@@ -44,10 +44,12 @@ func waitForTraefik() {
 		if err == nil {
 			completionTime, err := time.Parse(time.RFC3339, string(out))
 			if err == nil && time.Since(completionTime).Seconds() < 60 {
-				break
+				return
 			}
 		}
+		time.Sleep(1 * time.Second)
 	}
+	log.Error().Msg(L("Failed to reload K3s Traefik"))
 }
 
 // UninstallK3sTraefikConfig uninstall K3s Traefik configuration.