feature: use keepalvied to add vip for virtual cluster

cmd/kubenest/node-agent/app/client/client.go

@@ -60,7 +60,7 @@ var (
 	}
 	wg sync.WaitGroup
 
-	wsAddr    []string // websocket client connect address list
+	WsAddr    []string // websocket client connect address list
 	filePath  string   // the server path to save upload file
 	fileName  string   // local file to upload
 	params    []string // New slice to hold multiple command parameters
@@ -79,7 +79,7 @@ func cmdCheckRun(cmd *cobra.Command, args []string) error {
 	headers := http.Header{
 		"Authorization": {"Basic " + auth},
 	}
-	for _, addr := range wsAddr {
+	for _, addr := range WsAddr {
 		wg.Add(1)
 		go func(addr string) {
 			defer wg.Done()
@@ -106,15 +106,11 @@ func init() {
 	// #nosec G402
 	dialer.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 
-	ClientCmd.PersistentFlags().StringSliceVarP(&wsAddr, "addr", "a", []string{}, "WebSocket address (e.g., host1:port1,host2:port2)")
-	err := ClientCmd.MarkPersistentFlagRequired("addr")
-	if err != nil {
-		return
-	}
+	ClientCmd.PersistentFlags().StringSliceVarP(&WsAddr, "addr", "a", []string{}, "WebSocket address (e.g., host1:port1,host2:port2)")
 
 	// PreRunE check param
 	ClientCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
-		for _, value := range wsAddr {
+		for _, value := range WsAddr {
 			if _, exists := uniqueValuesMap[value]; exists {
 				return errors.New("duplicate values are not allowed")
 			}
@@ -137,7 +133,7 @@ func init() {
 	_ = uploadCmd.MarkFlagRequired("path")
 
 	ttyCmd.Flags().StringVarP(&operation, "operation", "o", "", "Operation to perform")
-	err = ttyCmd.MarkFlagRequired("operation") // Ensure 'operation' flag is required for ttyCmd
+	err := ttyCmd.MarkFlagRequired("operation") // Ensure 'operation' flag is required for ttyCmd
 	if err != nil {
 		return
 	}
@@ -157,7 +153,7 @@ func cmdTtyRun(cmd *cobra.Command, args []string) error {
 	}
 	cmdStr := fmt.Sprintf("command=%s", operation)
 	// execute one every wsAddr
-	for _, addr := range wsAddr {
+	for _, addr := range WsAddr {
 		wsURL := fmt.Sprintf("wss://%s/tty/?%s", addr, cmdStr)
 		fmt.Println("Executing tty:", cmdStr, "on", addr)
 		err := connectTty(wsURL, headers)
@@ -294,7 +290,7 @@ func executeWebSocketCommand(auth string) error {
 	}
 
 	// execute one every wsAddr
-	for _, addr := range wsAddr {
+	for _, addr := range WsAddr {
 		wg.Add(1)
 		go func(addr string) {
 			defer wg.Done()
@@ -314,7 +310,7 @@ func uploadFile(filePath, fileName, auth string) error {
 	headers := http.Header{
 		"Authorization": {"Basic " + auth},
 	}
-	for _, addr := range wsAddr {
+	for _, addr := range WsAddr {
 		wg.Add(1)
 		go func(addr string) {
 			defer wg.Done()

cmd/kubenest/node-agent/app/root.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"os"
+	"strings"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -33,6 +34,7 @@ func initConfig() {
 	currentDir, _ := os.Getwd()
 	viper.AddConfigPath(currentDir)
 	viper.AddConfigPath("/srv/node-agent/agent.env")
+	viper.SetConfigType("toml")
 	// If a agent.env file is found, read it in.
 	if err := viper.ReadInConfig(); err != nil {
 		log.Warnf("Load config file error, %s", err)
@@ -46,8 +48,27 @@ func initConfig() {
 	}
 }
 
+func initWebSocketAddr() {
+	err := viper.BindPFlag("ADDR", client.ClientCmd.PersistentFlags().Lookup("addr"))
+	if err != nil {
+		log.Fatalf("Failed to bind flag: %v", err)
+		return
+	}
+	err = viper.BindEnv("ADDR", "ADDR")
+	if err != nil {
+		log.Fatalf("Failed to bind env: %v", err)
+		return
+	}
+	// Initialize addr value from viper
+	log.Infof(strings.Join(viper.AllKeys(), ","))
+	if viper.Get("addr") != nil {
+		client.WsAddr = viper.GetStringSlice("addr")
+		log.Infof("addr: %v", client.WsAddr)
+	}
+}
+
 func init() {
-	cobra.OnInitialize(initConfig)
+	cobra.OnInitialize(initConfig, initWebSocketAddr)
 
 	RootCmd.PersistentFlags().StringVarP(&user, "user", "u", "", "Username for authentication")
 	RootCmd.PersistentFlags().StringVarP(&password, "password", "p", "", "Password for authentication")

deploy/crds/kosmos.io_virtualclusters.yaml

@@ -170,6 +170,10 @@ spec:
               updateTime:
                 format: date-time
                 type: string
+              vipMap:
+                additionalProperties:
+                  type: string
+                type: object
             type: object
         required:
         - spec

deploy/virtual-cluster-components-manifest-cm.yaml

@@ -4,6 +4,7 @@ data:
     [
       {"name": "kube-proxy", "path": "/kosmos/manifest/kube-proxy/*.yaml"},
       {"name": "calico", "path": "/kosmos/manifest/calico/*.yaml"},
+      {"name": "keepalived", "path": "/kosmos/manifest/keepalived/*.yaml"},
     ]
   host-core-dns-components: |
     [

deploy/virtual-cluster-vip-pool-cm.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kosmos-vip-pool
+  namespace: kosmos-system
+data:
+  vip-config.yaml: |
+    # can be use for vc, the ip formate is 192.168.0.1 and 192.168.0.2-192.168.0.10
+    vipPool:
+      - 192.168.0.1-192.168.0.10

hack/node-agent/init.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
-WEB_USER="$WEB_USER" sed -i 's/^WEB_USER=.*/WEB_USER='"$WEB_USER"'/' /app/agent.env
-WEB_PASS="$WEB_PASS" sed -i 's/^WEB_PASS=.*/WEB_PASS='"$WEB_PASS"'/' /app/agent.env
+WEB_USER="$WEB_USER" sed -i 's/^WEB_USER=.*/WEB_USER="'"$WEB_USER"'"/' /app/agent.env
+WEB_PASS="$WEB_PASS" sed -i 's/^WEB_PASS=.*/WEB_PASS="'"$WEB_PASS"'"/' /app/agent.env
 sha256sum /app/node-agent > /app/node-agent.sum
 sha256sum /host-path/node-agent >> /app/node-agent.sum
 rsync -avz /app/ /host-path/

pkg/apis/kosmos/v1alpha1/virtualcluster_types.go

@@ -112,6 +112,8 @@ type VirtualClusterStatus struct {
 	Port int32 `json:"port,omitempty"`
 	// +optional
 	PortMap map[string]int32 `json:"portMap,omitempty"`
+	// +optional
+	VipMap map[string]string `json:"vipMap,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/kubenest/constants/constant.go

@@ -100,6 +100,17 @@ const (
 	ApiServerNetworkProxyAdminPortKey  = "apiserver-network-proxy-admin-port"
 	VirtualClusterPortNum              = 5
 
+	// vip
+	VipPoolConfigMapName        = "kosmos-vip-pool"
+	VipPoolKey                  = "vip-config.yaml"
+	VcVipStatusKey              = "vip-key"
+	VipKeepAlivedNodeLabelKey   = "kosmos.io/keepalived-node"
+	VipKeepAlivedNodeLabelValue = "true"
+	VipKeepAlivedNodeRoleKey    = "kosmos.io/keepalived-role"
+	VipKeepAlivedNodeRoleMaster = "master"
+	VipKeepalivedNodeRoleBackup = "backup"
+	VipKeepAlivedReplicas       = 3
+
 	ManifestComponentsConfigMap = "components-manifest-cm"
 	NodePoolConfigmap           = "node-pool"
 	NodeVirtualclusterState     = "virtualcluster"