diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f36d59a..91e7bdd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,7 +54,7 @@ jobs: with: context: . push: true - tags: rancher/k3k:k3k-kubelet + tags: rancher/k3k:k3k-kubelet-dev file: package/Dockerfile.kubelet platforms: linux/amd64 diff --git a/charts/k3k/templates/deployment.yaml b/charts/k3k/templates/deployment.yaml index fa42807..9e749d9 100644 --- a/charts/k3k/templates/deployment.yaml +++ b/charts/k3k/templates/deployment.yaml @@ -19,9 +19,11 @@ spec: - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} name: {{ .Chart.Name }} - environment: + env: - name: CLUSTER_CIDR value: {{ .Values.host.clusterCIDR }} + - name: SHARED_AGENT_IMAGE + value: "{{ .Values.sharedAgent.image.repository }}:{{ .Values.sharedAgent.image.tag }}" ports: - containerPort: 8080 name: https diff --git a/charts/k3k/values.yaml b/charts/k3k/values.yaml index 224bdf3..4fcb08a 100644 --- a/charts/k3k/values.yaml +++ b/charts/k3k/values.yaml @@ -22,3 +22,9 @@ serviceAccount: # The name of the service account to use. # If not set and create is true, a name is generated using the fullname template name: "" + +# configuration related to the shared agent mode in k3k +sharedAgent: + image: + repository: "rancher/k3k" + tag: "k3k-kubelet-dev" \ No newline at end of file diff --git a/cli/cmds/cluster/create.go b/cli/cmds/cluster/create.go index bd2b9b8..6c95374 100644 --- a/cli/cmds/cluster/create.go +++ b/cli/cmds/cluster/create.go @@ -7,20 +7,18 @@ import ( "os" "path/filepath" "strings" - "time" "github.com/rancher/k3k/cli/cmds" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + "github.com/rancher/k3k/pkg/controller" "github.com/rancher/k3k/pkg/controller/cluster" "github.com/rancher/k3k/pkg/controller/cluster/server" "github.com/rancher/k3k/pkg/controller/kubeconfig" - "github.com/rancher/k3k/pkg/controller/util" "github.com/sirupsen/logrus" "github.com/urfave/cli" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/user" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/clientcmd" @@ -28,15 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var ( - Scheme = runtime.NewScheme() - backoff = wait.Backoff{ - Steps: 5, - Duration: 20 * time.Second, - Factor: 2, - Jitter: 0.1, - } -) +var Scheme = runtime.NewScheme() func init() { _ = clientgoscheme.AddToScheme(Scheme) @@ -120,7 +110,7 @@ var ( func create(clx *cli.Context) error { ctx := context.Background() - if err := validateCreateFlags(clx); err != nil { + if err := validateCreateFlags(); err != nil { return err } @@ -173,13 +163,13 @@ func create(clx *cli.Context) error { logrus.Infof("Extracting Kubeconfig for [%s] cluster", name) cfg := &kubeconfig.KubeConfig{ - CN: util.AdminCommonName, + CN: controller.AdminCommonName, ORG: []string{user.SystemPrivilegedGroup}, ExpiryDate: 0, } logrus.Infof("waiting for cluster to be available..") var kubeconfig []byte - if err := retry.OnError(backoff, apierrors.IsNotFound, func() error { + if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error { kubeconfig, err = cfg.Extract(ctx, ctrlClient, cluster, host[0]) if err != nil { return err @@ -203,7 +193,7 @@ func create(clx *cli.Context) error { return os.WriteFile(cluster.Name+"-kubeconfig.yaml", kubeconfig, 0644) } -func validateCreateFlags(clx *cli.Context) error { +func validateCreateFlags() error { if persistenceType != server.EphermalNodesType && persistenceType != server.DynamicNodesType { return errors.New("invalid persistence type") diff --git a/cli/cmds/kubeconfig/kubeconfig.go b/cli/cmds/kubeconfig/kubeconfig.go index a78e15d..d527abb 100644 --- a/cli/cmds/kubeconfig/kubeconfig.go +++ b/cli/cmds/kubeconfig/kubeconfig.go @@ -10,14 +10,13 @@ import ( "github.com/rancher/k3k/cli/cmds" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + "github.com/rancher/k3k/pkg/controller" "github.com/rancher/k3k/pkg/controller/kubeconfig" - "github.com/rancher/k3k/pkg/controller/util" "github.com/sirupsen/logrus" "github.com/urfave/cli" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/user" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/clientcmd" @@ -31,19 +30,13 @@ func init() { } var ( - Scheme = runtime.NewScheme() - name string - cn string - org cli.StringSlice - altNames cli.StringSlice - expirationDays int64 - configName string - backoff = wait.Backoff{ - Steps: 5, - Duration: 20 * time.Second, - Factor: 2, - Jitter: 0.1, - } + Scheme = runtime.NewScheme() + name string + cn string + org cli.StringSlice + altNames cli.StringSlice + expirationDays int64 + configName string generateKubeconfigFlags = []cli.Flag{ cli.StringFlag{ Name: "name", @@ -59,7 +52,7 @@ var ( Name: "cn", Usage: "Common name (CN) of the generated certificates for the kubeconfig", Destination: &cn, - Value: util.AdminCommonName, + Value: controller.AdminCommonName, }, cli.StringSliceFlag{ Name: "org", @@ -141,7 +134,7 @@ func generate(clx *cli.Context) error { } logrus.Infof("waiting for cluster to be available..") var kubeconfig []byte - if err := retry.OnError(backoff, apierrors.IsNotFound, func() error { + if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error { kubeconfig, err = cfg.Extract(ctx, ctrlClient, &cluster, host[0]) if err != nil { return err diff --git a/go.mod b/go.mod index 5651293..64204ef 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( go.etcd.io/etcd/api/v3 v3.5.14 go.etcd.io/etcd/client/v3 v3.5.14 go.uber.org/zap v1.26.0 + gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.31.1 k8s.io/apimachinery v0.31.1 k8s.io/apiserver v0.31.0 @@ -120,7 +121,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.29.2 // indirect k8s.io/component-base v0.29.2 // indirect k8s.io/kms v0.29.2 // indirect diff --git a/k3k-kubelet/config.go b/k3k-kubelet/config.go index e75ae87..c4beba1 100644 --- a/k3k-kubelet/config.go +++ b/k3k-kubelet/config.go @@ -9,52 +9,56 @@ import ( // Config has all virtual-kubelet startup options type config struct { - clusterName string `yaml:"clusterName"` - clusterNamespace string `yaml:"clusterNamespace"` - hostConfigPath string `yaml:"hostConfigPath"` - virtualConfigPath string `yaml:"virtualConfigPath"` - kubeletPort string `yaml:"kubeletPort"` - nodeName string `yaml:"nodeName"` - agentPodIP string `yaml:"agentPodIP"` - token string `yaml:"token"` + ClusterName string `yaml:"clusterName,omitempty"` + ClusterNamespace string `yaml:"clusterNamespace,omitempty"` + NodeName string `yaml:"nodeName,omitempty"` + Token string `yaml:"token,omitempty"` + AgentHostname string `yaml:"agentHostname,omitempty"` + HostConfigPath string `yaml:"hostConfigPath,omitempty"` + VirtualConfigPath string `yaml:"virtualConfigPath,omitempty"` + KubeletPort string `yaml:"kubeletPort,omitempty"` } -func (t *config) UnmarshalYAML(data []byte) error { +func (t *config) unmarshalYAML(data []byte) error { var c config + if err := yaml.Unmarshal(data, &c); err != nil { return err } - if t.clusterName == "" { - t.clusterName = c.clusterName + + if t.ClusterName == "" { + t.ClusterName = c.ClusterName + } + if t.ClusterNamespace == "" { + t.ClusterNamespace = c.ClusterNamespace } - if t.clusterNamespace == "" { - t.clusterNamespace = c.clusterNamespace + if t.HostConfigPath == "" { + t.HostConfigPath = c.HostConfigPath } - if t.hostConfigPath == "" { - t.hostConfigPath = c.hostConfigPath + if t.VirtualConfigPath == "" { + t.VirtualConfigPath = c.VirtualConfigPath } - if t.virtualConfigPath == "" { - t.virtualConfigPath = c.virtualConfigPath + if t.KubeletPort == "" { + t.KubeletPort = c.KubeletPort } - if t.kubeletPort == "" { - t.kubeletPort = c.kubeletPort + if t.AgentHostname == "" { + t.AgentHostname = c.AgentHostname } - if t.nodeName == "" { - t.nodeName = c.nodeName + if t.NodeName == "" { + t.NodeName = c.NodeName } - return nil } func (t *config) Validate() error { - if t.clusterName == "" { + if t.ClusterName == "" { return errors.New("cluster name is not provided") } - if t.clusterNamespace == "" { + if t.ClusterNamespace == "" { return errors.New("cluster namespace is not provided") } - if t.agentPodIP == "" { - return errors.New("agent POD IP is not provided") + if t.AgentHostname == "" { + return errors.New("agent Hostname is not provided") } return nil } @@ -68,5 +72,5 @@ func (t *config) Parse(path string) error { if err != nil { return err } - return t.UnmarshalYAML(configFileBytes) + return t.unmarshalYAML(configFileBytes) } diff --git a/k3k-kubelet/kubelet.go b/k3k-kubelet/kubelet.go index 98866c5..884d2a9 100644 --- a/k3k-kubelet/kubelet.go +++ b/k3k-kubelet/kubelet.go @@ -5,7 +5,6 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "net" "net/http" "os" "time" @@ -13,16 +12,16 @@ import ( certutil "github.com/rancher/dynamiclistener/cert" "github.com/rancher/k3k/k3k-kubelet/provider" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + "github.com/rancher/k3k/pkg/controller" + "github.com/rancher/k3k/pkg/controller/cluster/server" "github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap" "github.com/rancher/k3k/pkg/controller/kubeconfig" - "github.com/rancher/k3k/pkg/controller/util" "github.com/virtual-kubelet/virtual-kubelet/log" "github.com/virtual-kubelet/virtual-kubelet/node" "github.com/virtual-kubelet/virtual-kubelet/node/nodeutil" "go.uber.org/zap" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -33,15 +32,7 @@ import ( ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) -var ( - Scheme = runtime.NewScheme() - backoff = wait.Backoff{ - Steps: 5, - Duration: 5 * time.Second, - Factor: 2, - Jitter: 0.1, - } -) +var Scheme = runtime.NewScheme() func init() { _ = clientgoscheme.AddToScheme(Scheme) @@ -58,7 +49,7 @@ type kubelet struct { } func newKubelet(c *config) (*kubelet, error) { - hostConfig, err := clientcmd.BuildConfigFromFlags("", c.hostConfigPath) + hostConfig, err := clientcmd.BuildConfigFromFlags("", c.HostConfigPath) if err != nil { return nil, err } @@ -70,7 +61,7 @@ func newKubelet(c *config) (*kubelet, error) { return nil, err } - virtConfig, err := virtRestConfig(context.Background(), c.virtualConfigPath, hostClient, c.clusterName, c.clusterNamespace) + virtConfig, err := virtRestConfig(context.Background(), c.VirtualConfigPath, hostClient, c.ClusterName, c.ClusterNamespace) if err != nil { return nil, err } @@ -80,16 +71,16 @@ func newKubelet(c *config) (*kubelet, error) { return nil, err } return &kubelet{ - name: c.nodeName, + name: c.NodeName, hostConfig: hostConfig, hostClient: hostClient, virtClient: virtClient, }, nil } -func (k *kubelet) RegisterNode(srvPort, namespace, name, podIP string) error { - providerFunc := k.newProviderFunc(namespace, name, podIP) - nodeOpts := k.nodeOpts(srvPort, namespace, name, podIP) +func (k *kubelet) RegisterNode(srvPort, namespace, name, hostname string) error { + providerFunc := k.newProviderFunc(namespace, name, hostname) + nodeOpts := k.nodeOpts(srvPort, namespace, name, hostname) var err error k.node, err = nodeutil.NewNode(k.name, providerFunc, nodeutil.WithClient(k.virtClient), nodeOpts) @@ -129,7 +120,7 @@ func (k *kubelet) Start(ctx context.Context) { fmt.Printf("node exited without an error") } -func (k *kubelet) newProviderFunc(namespace, name, podIP string) nodeutil.NewProviderFunc { +func (k *kubelet) newProviderFunc(namespace, name, hostname string) nodeutil.NewProviderFunc { return func(pc nodeutil.ProviderConfig) (nodeutil.Provider, node.NodeProvider, error) { utilProvider, err := provider.New(*k.hostConfig, namespace, name) if err != nil { @@ -137,12 +128,12 @@ func (k *kubelet) newProviderFunc(namespace, name, podIP string) nodeutil.NewPro } nodeProvider := provider.Node{} - provider.ConfigureNode(pc.Node, podIP, k.port) + provider.ConfigureNode(pc.Node, hostname, k.port) return utilProvider, &nodeProvider, nil } } -func (k *kubelet) nodeOpts(srvPort, namespace, name, podIP string) nodeutil.NodeOpt { +func (k *kubelet) nodeOpts(srvPort, namespace, name, hostname string) nodeutil.NodeOpt { return func(c *nodeutil.NodeConfig) error { c.HTTPListenAddr = fmt.Sprintf(":%s", srvPort) // set up the routes @@ -155,7 +146,7 @@ func (k *kubelet) nodeOpts(srvPort, namespace, name, podIP string) nodeutil.Node ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - tlsConfig, err := loadTLSConfig(ctx, k.hostClient, name, namespace, k.name, podIP) + tlsConfig, err := loadTLSConfig(ctx, k.hostClient, name, namespace, k.name, hostname) if err != nil { return fmt.Errorf("unable to get tls config: %w", err) } @@ -164,18 +155,18 @@ func (k *kubelet) nodeOpts(srvPort, namespace, name, podIP string) nodeutil.Node } } -func virtRestConfig(ctx context.Context, virtualConfigPath string, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace string) (*rest.Config, error) { - if virtualConfigPath != "" { - return clientcmd.BuildConfigFromFlags("", virtualConfigPath) +func virtRestConfig(ctx context.Context, VirtualConfigPath string, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace string) (*rest.Config, error) { + if VirtualConfigPath != "" { + return clientcmd.BuildConfigFromFlags("", VirtualConfigPath) } // virtual kubeconfig file is empty, trying to fetch the k3k cluster kubeconfig var cluster v1alpha1.Cluster if err := hostClient.Get(ctx, types.NamespacedName{Namespace: clusterNamespace, Name: clusterName}, &cluster); err != nil { return nil, err } - endpoint := fmt.Sprintf("%s.%s", util.ServerSvcName(&cluster), util.ClusterNamespace(&cluster)) + endpoint := fmt.Sprintf("%s.%s", server.ServiceName(cluster.Name), cluster.Namespace) var b *bootstrap.ControlRuntimeBootstrap - if err := retry.OnError(backoff, func(err error) bool { + if err := retry.OnError(controller.Backoff, func(err error) bool { return err == nil }, func() error { var err error @@ -185,7 +176,7 @@ func virtRestConfig(ctx context.Context, virtualConfigPath string, hostClient ct return nil, fmt.Errorf("unable to decode bootstrap: %w", err) } adminCert, adminKey, err := kubeconfig.CreateClientCertKey( - util.AdminCommonName, []string{user.SystemPrivilegedGroup}, + controller.AdminCommonName, []string{user.SystemPrivilegedGroup}, nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, time.Hour*24*time.Duration(356), b.ClientCA.Content, b.ClientCAKey.Content) @@ -193,7 +184,7 @@ func virtRestConfig(ctx context.Context, virtualConfigPath string, hostClient ct return nil, err } - url := fmt.Sprintf("https://%s:%d", util.ServerSvcName(&cluster), util.ServerPort) + url := fmt.Sprintf("https://%s:%d", server.ServiceName(cluster.Name), server.ServerPort) kubeconfigData, err := kubeconfigBytes(url, []byte(b.ServerCA.Content), adminCert, adminKey) if err != nil { return nil, err @@ -229,7 +220,7 @@ func kubeconfigBytes(url string, serverCA, clientCert, clientKey []byte) ([]byte return kubeconfig, nil } -func loadTLSConfig(ctx context.Context, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace, nodeName, ipStr string) (*tls.Config, error) { +func loadTLSConfig(ctx context.Context, hostClient ctrlruntimeclient.Client, clusterName, clusterNamespace, nodeName, hostname string) (*tls.Config, error) { var ( cluster v1alpha1.Cluster b *bootstrap.ControlRuntimeBootstrap @@ -237,8 +228,8 @@ func loadTLSConfig(ctx context.Context, hostClient ctrlruntimeclient.Client, clu if err := hostClient.Get(ctx, types.NamespacedName{Name: clusterName, Namespace: clusterNamespace}, &cluster); err != nil { return nil, err } - endpoint := fmt.Sprintf("%s.%s", util.ServerSvcName(&cluster), util.ClusterNamespace(&cluster)) - if err := retry.OnError(backoff, func(err error) bool { + endpoint := fmt.Sprintf("%s.%s", server.ServiceName(cluster.Name), cluster.Namespace) + if err := retry.OnError(controller.Backoff, func(err error) bool { return err != nil }, func() error { var err error @@ -248,7 +239,7 @@ func loadTLSConfig(ctx context.Context, hostClient ctrlruntimeclient.Client, clu return nil, fmt.Errorf("unable to decode bootstrap: %w", err) } altNames := certutil.AltNames{ - IPs: []net.IP{net.ParseIP(ipStr)}, + DNSNames: []string{hostname}, } cert, key, err := kubeconfig.CreateClientCertKey(nodeName, nil, &altNames, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, 0, b.ServerCA.Content, b.ServerCAKey.Content) if err != nil { diff --git a/k3k-kubelet/main.go b/k3k-kubelet/main.go index 9138819..a6ddf84 100644 --- a/k3k-kubelet/main.go +++ b/k3k-kubelet/main.go @@ -22,45 +22,45 @@ func main() { cli.StringFlag{ Name: "cluster-name", Usage: "Name of the k3k cluster", - Destination: &c.clusterName, + Destination: &c.ClusterName, EnvVar: "CLUSTER_NAME", }, cli.StringFlag{ Name: "cluster-namespace", Usage: "Namespace of the k3k cluster", - Destination: &c.clusterNamespace, + Destination: &c.ClusterNamespace, EnvVar: "CLUSTER_NAMESPACE", }, cli.StringFlag{ Name: "cluster-token", Usage: "K3S token of the k3k cluster", - Destination: &c.token, - EnvVar: "CLUSTER_Token", + Destination: &c.Token, + EnvVar: "CLUSTER_TOKEN", }, cli.StringFlag{ Name: "host-config-path", Usage: "Path to the host kubeconfig, if empty then virtual-kubelet will use incluster config", - Destination: &c.hostConfigPath, + Destination: &c.HostConfigPath, EnvVar: "HOST_KUBECONFIG", }, cli.StringFlag{ Name: "virtual-config-path", Usage: "Path to the k3k cluster kubeconfig, if empty then virtual-kubelet will create its own config from k3k cluster", - Destination: &c.virtualConfigPath, + Destination: &c.VirtualConfigPath, EnvVar: "CLUSTER_NAME", }, cli.StringFlag{ Name: "kubelet-port", Usage: "kubelet API port number", - Destination: &c.kubeletPort, + Destination: &c.KubeletPort, EnvVar: "SERVER_PORT", Value: "9443", }, cli.StringFlag{ - Name: "agent-pod-ip", - Usage: "Agent Pod IP used for TLS SAN for the kubelet server", - Destination: &c.agentPodIP, - EnvVar: "AGENT_POD_IP", + Name: "agent-hostname", + Usage: "Agent Hostname used for TLS SAN for the kubelet server", + Destination: &c.AgentHostname, + EnvVar: "AGENT_HOSTNAME", }, cli.StringFlag{ Name: "config", @@ -92,7 +92,7 @@ func Run(clx *cli.Context) { os.Exit(-1) } - if err := k.RegisterNode(c.kubeletPort, c.clusterNamespace, c.clusterName, c.agentPodIP); err != nil { + if err := k.RegisterNode(c.KubeletPort, c.ClusterNamespace, c.ClusterName, c.AgentHostname); err != nil { fmt.Printf("failed to register new node: %v", err) os.Exit(-1) } diff --git a/k3k-kubelet/provider/configure.go b/k3k-kubelet/provider/configure.go index 44db2f5..45d93a4 100644 --- a/k3k-kubelet/provider/configure.go +++ b/k3k-kubelet/provider/configure.go @@ -6,13 +6,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func ConfigureNode(node *v1.Node, podIP string, servicePort int) { +func ConfigureNode(node *v1.Node, hostname string, servicePort int) { node.Status.Conditions = nodeConditions() node.Status.DaemonEndpoints.KubeletEndpoint.Port = int32(servicePort) node.Status.Addresses = []v1.NodeAddress{ { - Type: v1.NodeExternalIP, - Address: podIP, + Type: v1.NodeHostName, + Address: hostname, }, } node.Status.Capacity = v1.ResourceList{ diff --git a/k3k-kubelet/provider/provider.go b/k3k-kubelet/provider/provider.go index f836dcc..e8fbf86 100644 --- a/k3k-kubelet/provider/provider.go +++ b/k3k-kubelet/provider/provider.go @@ -2,15 +2,13 @@ package provider import ( "context" - "crypto/sha256" - "encoding/hex" "fmt" "io" "net/http" "strconv" - "strings" dto "github.com/prometheus/client_model/go" + "github.com/rancher/k3k/pkg/controller" "github.com/virtual-kubelet/virtual-kubelet/node/api" "github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1" "go.uber.org/zap" @@ -313,23 +311,5 @@ func (p *Provider) translateFrom(hostPod *corev1.Pod) *corev1.Pod { } func (p *Provider) hostName(virtualNamespace string, virtualName string) string { - return safeConcatName(p.ClusterName, p.ClusterNamespace, virtualNamespace, virtualName) -} - -// safeConcatName concatenates the given strings and ensures the returned name is under 64 characters -// by cutting the string off at 57 characters and setting the last 6 with an encoded version of the concatenated string. -func safeConcatName(name ...string) string { - fullPath := strings.Join(name, "-") - if len(fullPath) < 64 { - return fullPath - } - digest := sha256.Sum256([]byte(fullPath)) - // since we cut the string in the middle, the last char may not be compatible with what is expected in k8s - // we are checking and if necessary removing the last char - c := fullPath[56] - if 'a' <= c && c <= 'z' || '0' <= c && c <= '9' { - return fullPath[0:57] + "-" + hex.EncodeToString(digest[0:])[0:5] - } - - return fullPath[0:56] + "-" + hex.EncodeToString(digest[0:])[0:6] + return controller.SafeConcatName(p.ClusterName, p.ClusterNamespace, virtualNamespace, virtualName) } diff --git a/main.go b/main.go index 35e32b8..8388440 100644 --- a/main.go +++ b/main.go @@ -27,10 +27,11 @@ const ( ) var ( - scheme = runtime.NewScheme() - clusterCIDR string - kubeconfig string - flags = []cli.Flag{ + scheme = runtime.NewScheme() + clusterCIDR string + sharedAgentImage string + kubeconfig string + flags = []cli.Flag{ cli.StringFlag{ Name: "kubeconfig", EnvVar: "KUBECONFIG", @@ -42,7 +43,15 @@ var ( EnvVar: "CLUSTER_CIDR", Usage: "Cluster CIDR to be added to the networkpolicy of the clustersets", Destination: &clusterCIDR, - }} + }, + cli.StringFlag{ + Name: "shared-agent-image", + EnvVar: "SHARED_AGENT_IMAGE", + Usage: "K3K Virtual Kubelet image ", + Value: "rancher/k3k:k3k-kubelet-dev", + Destination: &sharedAgentImage, + }, + } ) func init() { @@ -77,8 +86,7 @@ func run(clx *cli.Context) error { if err != nil { return fmt.Errorf("Failed to create new controller runtime manager: %v", err) } - - if err := cluster.Add(ctx, mgr); err != nil { + if err := cluster.Add(ctx, mgr, sharedAgentImage); err != nil { return fmt.Errorf("Failed to add the new cluster controller: %v", err) } diff --git a/pkg/controller/cluster/agent/agent.go b/pkg/controller/cluster/agent/agent.go index e849f6c..95d5771 100644 --- a/pkg/controller/cluster/agent/agent.go +++ b/pkg/controller/cluster/agent/agent.go @@ -2,19 +2,29 @@ package agent import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" - "github.com/rancher/k3k/pkg/controller/cluster/config" + "github.com/rancher/k3k/pkg/controller" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + configName = "agent-config" +) + type Agent interface { + Name() string Config() (ctrlruntimeclient.Object, error) Resources() ([]ctrlruntimeclient.Object, error) } -func New(cluster *v1alpha1.Cluster, serviceIP string) Agent { - if cluster.Spec.Mode == config.VirtualNodeMode { +func New(cluster *v1alpha1.Cluster, serviceIP, sharedAgentImage string) Agent { + if cluster.Spec.Mode == VirtualNodeMode { return NewVirtualAgent(cluster, serviceIP) } else { - return NewSharedAgent(cluster, serviceIP) + return NewSharedAgent(cluster, serviceIP, sharedAgentImage) } } + +func configSecretName(clusterName string) string { + return controller.ObjectName(clusterName, nil, configName) +} diff --git a/pkg/controller/cluster/agent/shared.go b/pkg/controller/cluster/agent/shared.go index 39dbcb3..b6e7251 100644 --- a/pkg/controller/cluster/agent/shared.go +++ b/pkg/controller/cluster/agent/shared.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" - "github.com/rancher/k3k/pkg/controller/util" + "github.com/rancher/k3k/pkg/controller" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -13,19 +13,22 @@ import ( ) const ( - virtualKubeletImage = "rancher/k3k:k3k-kubelet" - virtualKubeletConfigPath = "/opt/rancher/k3k/config.yaml" + sharedKubeletConfigPath = "/opt/rancher/k3k/config.yaml" + sharedNodeAgentName = "kubelet" + SharedNodeMode = "shared" ) type SharedAgent struct { - cluster *v1alpha1.Cluster - serviceIP string + cluster *v1alpha1.Cluster + serviceIP string + sharedAgentImage string } -func NewSharedAgent(cluster *v1alpha1.Cluster, serviceIP string) Agent { +func NewSharedAgent(cluster *v1alpha1.Cluster, serviceIP, sharedAgentImage string) Agent { return &SharedAgent{ - cluster: cluster, - serviceIP: serviceIP, + cluster: cluster, + serviceIP: serviceIP, + sharedAgentImage: sharedAgentImage, } } @@ -38,8 +41,8 @@ func (s *SharedAgent) Config() (ctrlruntimeclient.Object, error) { APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: util.AgentConfigName(s.cluster), - Namespace: util.ClusterNamespace(s.cluster), + Name: configSecretName(s.cluster.Name), + Namespace: s.cluster.Namespace, }, Data: map[string][]byte{ "config.yaml": []byte(config), @@ -51,12 +54,13 @@ func sharedAgentData(cluster *v1alpha1.Cluster) string { return fmt.Sprintf(`clusterName: %s clusterNamespace: %s nodeName: %s -token: %s`, cluster.Name, cluster.Namespace, cluster.Name+"-"+"k3k-kubelet", cluster.Spec.Token) +agentHostname: %s +token: %s`, cluster.Name, cluster.Namespace, cluster.Name+"-"+"k3k-kubelet", cluster.Name+"-"+"k3k-kubelet", cluster.Spec.Token) } func (s *SharedAgent) Resources() ([]ctrlruntimeclient.Object, error) { var objs []ctrlruntimeclient.Object - objs = append(objs, s.serviceAccount(), s.role(), s.roleBinding(), s.deployment()) + objs = append(objs, s.serviceAccount(), s.role(), s.roleBinding(), s.service(), s.deployment()) return objs, nil } @@ -68,7 +72,7 @@ func (s *SharedAgent) deployment() *apps.Deployment { "mode": "shared", }, } - name := s.cluster.Name + "-" + "k3k-kubelet" + name := s.Name() return &apps.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", @@ -76,7 +80,7 @@ func (s *SharedAgent) deployment() *apps.Deployment { }, ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: util.ClusterNamespace(s.cluster), + Namespace: s.cluster.Namespace, Labels: selector.MatchLabels, }, Spec: apps.DeploymentSpec{ @@ -85,14 +89,14 @@ func (s *SharedAgent) deployment() *apps.Deployment { ObjectMeta: metav1.ObjectMeta{ Labels: selector.MatchLabels, }, - Spec: s.podSpec(virtualKubeletImage, name, &selector), + Spec: s.podSpec(s.sharedAgentImage, name, &selector), }, }, } } func (s *SharedAgent) podSpec(image, name string, affinitySelector *metav1.LabelSelector) v1.PodSpec { - args := []string{"--config", virtualKubeletConfigPath} + args := []string{"--config", sharedKubeletConfigPath} var limit v1.ResourceList return v1.PodSpec{ Affinity: &v1.Affinity{ @@ -105,13 +109,13 @@ func (s *SharedAgent) podSpec(image, name string, affinitySelector *metav1.Label }, }, }, - ServiceAccountName: s.cluster.Name + "-" + "k3k-kubelet", + ServiceAccountName: s.Name(), Volumes: []v1.Volume{ { Name: "config", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: util.AgentConfigName(s.cluster), + SecretName: configSecretName(s.cluster.Name), Items: []v1.KeyToPath{ { Key: "config.yaml", @@ -131,16 +135,6 @@ func (s *SharedAgent) podSpec(image, name string, affinitySelector *metav1.Label Limits: limit, }, Args: args, - Env: []v1.EnvVar{ - { - Name: "AGENT_POD_IP", - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "status.podIP", - }, - }, - }, - }, VolumeMounts: []v1.VolumeMount{ { Name: "config", @@ -152,6 +146,34 @@ func (s *SharedAgent) podSpec(image, name string, affinitySelector *metav1.Label }} } +func (s *SharedAgent) service() *v1.Service { + return &v1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: s.Name(), + Namespace: s.cluster.Namespace, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + Selector: map[string]string{ + "cluster": s.cluster.Name, + "type": "agent", + "mode": "shared", + }, + Ports: []v1.ServicePort{ + { + Name: "k3s-kubelet-port", + Protocol: v1.ProtocolTCP, + Port: 9443, + }, + }, + }, + } +} + func (s *SharedAgent) serviceAccount() *v1.ServiceAccount { return &v1.ServiceAccount{ TypeMeta: metav1.TypeMeta{ @@ -159,8 +181,8 @@ func (s *SharedAgent) serviceAccount() *v1.ServiceAccount { APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: s.cluster.Name + "-" + "k3k-kubelet", - Namespace: util.ClusterNamespace(s.cluster), + Name: s.Name(), + Namespace: s.cluster.Namespace, }, } } @@ -172,8 +194,8 @@ func (s *SharedAgent) role() *rbacv1.Role { APIVersion: "rbac.authorization.k8s.io/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: s.cluster.Name + "-" + "k3k-kubelet", - Namespace: util.ClusterNamespace(s.cluster), + Name: s.Name(), + Namespace: s.cluster.Namespace, }, Rules: []rbacv1.PolicyRule{ { @@ -202,20 +224,24 @@ func (s *SharedAgent) roleBinding() *rbacv1.RoleBinding { APIVersion: "rbac.authorization.k8s.io/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: s.cluster.Name + "-" + "k3k-kubelet", - Namespace: util.ClusterNamespace(s.cluster), + Name: s.Name(), + Namespace: s.cluster.Namespace, }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind: "Role", - Name: s.cluster.Name + "-" + "k3k-kubelet", + Name: s.Name(), }, Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: s.cluster.Name + "-" + "k3k-kubelet", - Namespace: util.ClusterNamespace(s.cluster), + Name: s.Name(), + Namespace: s.cluster.Namespace, }, }, } } + +func (s *SharedAgent) Name() string { + return controller.ObjectName(s.cluster.Name, nil, sharedNodeAgentName) +} diff --git a/pkg/controller/cluster/agent/virtual.go b/pkg/controller/cluster/agent/virtual.go index 05d12ec..28dd395 100644 --- a/pkg/controller/cluster/agent/virtual.go +++ b/pkg/controller/cluster/agent/virtual.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" - "github.com/rancher/k3k/pkg/controller/util" + "github.com/rancher/k3k/pkg/controller" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -12,6 +12,11 @@ import ( ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + VirtualNodeMode = "virtual" + virtualNodeAgentName = "kubelet" +) + type VirtualAgent struct { cluster *v1alpha1.Cluster serviceIP string @@ -33,8 +38,8 @@ func (v *VirtualAgent) Config() (ctrlruntimeclient.Object, error) { APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: util.AgentConfigName(v.cluster), - Namespace: util.ClusterNamespace(v.cluster), + Name: configSecretName(v.cluster.Name), + Namespace: v.cluster.Namespace, }, Data: map[string][]byte{ "config.yaml": []byte(config), @@ -55,7 +60,7 @@ with-node-id: true`, serviceIP, token) } func (v *VirtualAgent) deployment() *apps.Deployment { - image := util.K3SImage(v.cluster) + image := controller.K3SImage(v.cluster) const name = "k3k-agent" selector := metav1.LabelSelector{ @@ -71,8 +76,8 @@ func (v *VirtualAgent) deployment() *apps.Deployment { APIVersion: "apps/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: v.cluster.Name + "-" + name, - Namespace: util.ClusterNamespace(v.cluster), + Name: v.Name(), + Namespace: v.cluster.Namespace, Labels: selector.MatchLabels, }, Spec: apps.DeploymentSpec{ @@ -107,7 +112,7 @@ func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelect Name: "config", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: util.AgentConfigName(v.cluster), + SecretName: configSecretName(v.cluster.Name), Items: []v1.KeyToPath{ { Key: "config.yaml", @@ -212,3 +217,7 @@ func (v *VirtualAgent) podSpec(image, name string, args []string, affinitySelect return podSpec } + +func (v *VirtualAgent) Name() string { + return controller.ObjectName(v.cluster.Name, nil, virtualNodeAgentName) +} diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index 77023ed..8df3002 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -8,11 +8,10 @@ import ( "time" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + k3kcontroller "github.com/rancher/k3k/pkg/controller" "github.com/rancher/k3k/pkg/controller/cluster/agent" - "github.com/rancher/k3k/pkg/controller/cluster/config" "github.com/rancher/k3k/pkg/controller/cluster/server" "github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap" - "github.com/rancher/k3k/pkg/controller/util" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,6 +27,7 @@ import ( ) const ( + namePrefix = "k3k" clusterController = "k3k-cluster-controller" clusterFinalizerName = "cluster.k3k.io/finalizer" etcdPodFinalizerName = "etcdpod.k3k.io/finalizer" @@ -42,16 +42,18 @@ const ( ) type ClusterReconciler struct { - Client ctrlruntimeclient.Client - Scheme *runtime.Scheme + Client ctrlruntimeclient.Client + Scheme *runtime.Scheme + SharedAgentImage string } // Add adds a new controller to the manager -func Add(ctx context.Context, mgr manager.Manager) error { +func Add(ctx context.Context, mgr manager.Manager, sharedAgentImage string) error { // initialize a new Reconciler reconciler := ClusterReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SharedAgentImage: sharedAgentImage, } return ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.Cluster{}). @@ -76,20 +78,20 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request if !controllerutil.ContainsFinalizer(&cluster, clusterFinalizerName) { controllerutil.AddFinalizer(&cluster, clusterFinalizerName) if err := c.Client.Update(ctx, &cluster); err != nil { - return reconcile.Result{}, util.LogAndReturnErr("failed to add cluster finalizer", err) + return reconcile.Result{}, k3kcontroller.LogAndReturnErr("failed to add cluster finalizer", err) } } klog.Infof("enqueue cluster [%s]", cluster.Name) if err := c.createCluster(ctx, &cluster); err != nil { - return reconcile.Result{}, util.LogAndReturnErr("failed to create cluster", err) + return reconcile.Result{}, k3kcontroller.LogAndReturnErr("failed to create cluster", err) } return reconcile.Result{}, nil } // remove finalizer from the server pods and update them. matchingLabels := ctrlruntimeclient.MatchingLabels(map[string]string{"role": "server"}) - listOpts := &ctrlruntimeclient.ListOptions{Namespace: util.ClusterNamespace(&cluster)} + listOpts := &ctrlruntimeclient.ListOptions{Namespace: cluster.Namespace} matchingLabels.ApplyToList(listOpts) if err := c.Client.List(ctx, &podList, listOpts); err != nil { @@ -99,7 +101,7 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request if controllerutil.ContainsFinalizer(&pod, etcdPodFinalizerName) { controllerutil.RemoveFinalizer(&pod, etcdPodFinalizerName) if err := c.Client.Update(ctx, &pod); err != nil { - return reconcile.Result{}, util.LogAndReturnErr("failed to remove etcd finalizer", err) + return reconcile.Result{}, k3kcontroller.LogAndReturnErr("failed to remove etcd finalizer", err) } } } @@ -108,7 +110,7 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request // remove finalizer from the cluster and update it. controllerutil.RemoveFinalizer(&cluster, clusterFinalizerName) if err := c.Client.Update(ctx, &cluster); err != nil { - return reconcile.Result{}, util.LogAndReturnErr("failed to remove cluster finalizer", err) + return reconcile.Result{}, k3kcontroller.LogAndReturnErr("failed to remove cluster finalizer", err) } } klog.Infof("deleting cluster [%s]", cluster.Name) @@ -131,7 +133,7 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1 } } if err := c.Client.Update(ctx, cluster); err != nil { - return util.LogAndReturnErr("failed to update cluster with persistence type", err) + return k3kcontroller.LogAndReturnErr("failed to update cluster with persistence type", err) } cluster.Status.ClusterCIDR = cluster.Spec.ClusterCIDR @@ -147,32 +149,32 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1 klog.Infof("creating cluster service") serviceIP, err := c.createClusterService(ctx, cluster, s) if err != nil { - return util.LogAndReturnErr("failed to create cluster service", err) + return k3kcontroller.LogAndReturnErr("failed to create cluster service", err) } - if err := c.createClusterConfigs(ctx, cluster, serviceIP); err != nil { - return util.LogAndReturnErr("failed to create cluster configs", err) + if err := c.createClusterConfigs(ctx, cluster, s, serviceIP); err != nil { + return k3kcontroller.LogAndReturnErr("failed to create cluster configs", err) } // creating statefulsets in case the user chose a persistence type other than ephermal if err := c.server(ctx, cluster, s); err != nil { - return util.LogAndReturnErr("failed to create servers", err) + return k3kcontroller.LogAndReturnErr("failed to create servers", err) } if err := c.agent(ctx, cluster, serviceIP); err != nil { - return util.LogAndReturnErr("failed to create agents", err) + return k3kcontroller.LogAndReturnErr("failed to create agents", err) } if cluster.Spec.Expose != nil { if cluster.Spec.Expose.Ingress != nil { serverIngress, err := s.Ingress(ctx, c.Client) if err != nil { - return util.LogAndReturnErr("failed to create ingress object", err) + return k3kcontroller.LogAndReturnErr("failed to create ingress object", err) } if err := c.Client.Create(ctx, serverIngress); err != nil { if !apierrors.IsAlreadyExists(err) { - return util.LogAndReturnErr("failed to create server ingress", err) + return k3kcontroller.LogAndReturnErr("failed to create server ingress", err) } } } @@ -180,21 +182,21 @@ func (c *ClusterReconciler) createCluster(ctx context.Context, cluster *v1alpha1 bootstrapSecret, err := bootstrap.Generate(ctx, cluster, serviceIP) if err != nil { - return util.LogAndReturnErr("failed to generate new kubeconfig", err) + return k3kcontroller.LogAndReturnErr("failed to generate new kubeconfig", err) } if err := c.Client.Create(ctx, bootstrapSecret); err != nil { if !apierrors.IsAlreadyExists(err) { - return util.LogAndReturnErr("failed to create kubeconfig secret", err) + return k3kcontroller.LogAndReturnErr("failed to create kubeconfig secret", err) } } return c.Client.Update(ctx, cluster) } -func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v1alpha1.Cluster, serviceIP string) error { +func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server, serviceIP string) error { // create init node config - initServerConfig, err := config.Server(cluster, true, serviceIP) + initServerConfig, err := server.Config(true, serviceIP) if err != nil { return err } @@ -210,7 +212,7 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v } // create servers configuration - serverConfig, err := config.Server(cluster, false, serviceIP) + serverConfig, err := server.Config(false, serviceIP) if err != nil { return err } @@ -226,9 +228,9 @@ func (c *ClusterReconciler) createClusterConfigs(ctx context.Context, cluster *v return nil } -func (c *ClusterReconciler) createClusterService(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server) (string, error) { +func (c *ClusterReconciler) createClusterService(ctx context.Context, cluster *v1alpha1.Cluster, s *server.Server) (string, error) { // create cluster service - clusterService := server.Service(cluster) + clusterService := s.Service(cluster) if err := controllerutil.SetControllerReference(cluster, clusterService, c.Scheme); err != nil { return "", err @@ -242,8 +244,8 @@ func (c *ClusterReconciler) createClusterService(ctx context.Context, cluster *v var service v1.Service objKey := ctrlruntimeclient.ObjectKey{ - Namespace: util.ClusterNamespace(cluster), - Name: util.ServerSvcName(cluster), + Namespace: cluster.Namespace, + Name: server.ServiceName(cluster.Name), } if err := c.Client.Get(ctx, objKey, &service); err != nil { return "", err @@ -254,7 +256,7 @@ func (c *ClusterReconciler) createClusterService(ctx context.Context, cluster *v func (c *ClusterReconciler) server(ctx context.Context, cluster *v1alpha1.Cluster, server *server.Server) error { // create headless service for the statefulset - serverStatefulService := server.StatefulServerService(cluster) + serverStatefulService := server.StatefulServerService() if err := controllerutil.SetControllerReference(cluster, serverStatefulService, c.Scheme); err != nil { return err } @@ -263,7 +265,7 @@ func (c *ClusterReconciler) server(ctx context.Context, cluster *v1alpha1.Cluste return err } } - ServerStatefulSet, err := server.StatefulServer(ctx, cluster) + ServerStatefulSet, err := server.StatefulServer(ctx) if err != nil { return err } @@ -280,13 +282,12 @@ func (c *ClusterReconciler) server(ctx context.Context, cluster *v1alpha1.Cluste } func (c *ClusterReconciler) agent(ctx context.Context, cluster *v1alpha1.Cluster, serviceIP string) error { - agent := agent.New(cluster, serviceIP) + agent := agent.New(cluster, serviceIP, c.SharedAgentImage) agentsConfig, err := agent.Config() if err != nil { return err } - agentResources, err := agent.Resources() if err != nil { return err diff --git a/pkg/controller/cluster/pod.go b/pkg/controller/cluster/pod.go index 77c9c7c..f087869 100644 --- a/pkg/controller/cluster/pod.go +++ b/pkg/controller/cluster/pod.go @@ -11,9 +11,10 @@ import ( certutil "github.com/rancher/dynamiclistener/cert" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + k3kcontroller "github.com/rancher/k3k/pkg/controller" + "github.com/rancher/k3k/pkg/controller/cluster/server" "github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap" "github.com/rancher/k3k/pkg/controller/kubeconfig" - "github.com/rancher/k3k/pkg/controller/util" "github.com/sirupsen/logrus" "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" clientv3 "go.etcd.io/etcd/client/v3" @@ -60,11 +61,13 @@ func AddPodController(ctx context.Context, mgr manager.Manager) error { } func (p *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - s := strings.Split(req.Namespace, "-") - if len(s) <= 1 { - return reconcile.Result{}, util.LogAndReturnErr("failed to get cluster namespace", nil) + s := strings.Split(req.Name, "-") + if len(s) < 1 { + return reconcile.Result{}, k3kcontroller.LogAndReturnErr("failed to get cluster namespace", nil) + } + if s[0] != "k3k" { + return reconcile.Result{}, nil } - clusterName := s[1] var cluster v1alpha1.Cluster if err := p.Client.Get(ctx, types.NamespacedName{Name: clusterName}, &cluster); err != nil { @@ -83,7 +86,7 @@ func (p *PodReconciler) Reconcile(ctx context.Context, req reconcile.Request) (r for _, pod := range podList.Items { klog.Infof("Handle etcd server pod [%s/%s]", pod.Namespace, pod.Name) if err := p.handleServerPod(ctx, cluster, &pod); err != nil { - return reconcile.Result{}, util.LogAndReturnErr("failed to handle etcd pod", err) + return reconcile.Result{}, k3kcontroller.LogAndReturnErr("failed to handle etcd pod", err) } } return reconcile.Result{}, nil @@ -116,7 +119,7 @@ func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cl // remove server from etcd client, err := clientv3.New(clientv3.Config{ Endpoints: []string{ - fmt.Sprintf("https://%s.%s:2379", util.ServerSvcName(&cluster), pod.Namespace), + fmt.Sprintf("https://%s.%s:2379", server.ServiceName(cluster.Name), pod.Namespace), }, TLS: tlsConfig, }) @@ -146,9 +149,9 @@ func (p *PodReconciler) handleServerPod(ctx context.Context, cluster v1alpha1.Cl func (p *PodReconciler) getETCDTLS(cluster *v1alpha1.Cluster) (*tls.Config, error) { klog.Infof("generating etcd TLS client certificate for cluster [%s]", cluster.Name) token := cluster.Spec.Token - endpoint := fmt.Sprintf("%s.%s", util.ServerSvcName(cluster), util.ClusterNamespace(cluster)) + endpoint := fmt.Sprintf("%s.%s", server.ServiceName(cluster.Name), cluster.Namespace) var b *bootstrap.ControlRuntimeBootstrap - if err := retry.OnError(retry.DefaultBackoff, func(err error) bool { + if err := retry.OnError(k3kcontroller.Backoff, func(err error) bool { return true }, func() error { var err error diff --git a/pkg/controller/cluster/server/bootstrap/bootstrap.go b/pkg/controller/cluster/server/bootstrap/bootstrap.go index a8372f4..41f2285 100644 --- a/pkg/controller/cluster/server/bootstrap/bootstrap.go +++ b/pkg/controller/cluster/server/bootstrap/bootstrap.go @@ -9,7 +9,7 @@ import ( "time" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" - "github.com/rancher/k3k/pkg/controller/util" + "github.com/rancher/k3k/pkg/controller" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" @@ -36,7 +36,7 @@ func Generate(ctx context.Context, cluster *v1alpha1.Cluster, ip string) (*v1.Se token := cluster.Spec.Token var bootstrap *ControlRuntimeBootstrap - if err := retry.OnError(retry.DefaultBackoff, func(err error) bool { + if err := retry.OnError(controller.Backoff, func(err error) bool { return true }, func() error { var err error @@ -60,8 +60,8 @@ func Generate(ctx context.Context, cluster *v1alpha1.Cluster, ip string) (*v1.Se APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: cluster.Name + "-bootstrap", - Namespace: util.ClusterNamespace(cluster), + Name: controller.ObjectName(cluster.Name, nil, "bootstrap"), + Namespace: cluster.Namespace, OwnerReferences: []metav1.OwnerReference{ { APIVersion: cluster.APIVersion, diff --git a/pkg/controller/cluster/config/server.go b/pkg/controller/cluster/server/config.go similarity index 66% rename from pkg/controller/cluster/config/server.go rename to pkg/controller/cluster/server/config.go index 3d563c3..5e0bf87 100644 --- a/pkg/controller/cluster/config/server.go +++ b/pkg/controller/cluster/server/config.go @@ -1,35 +1,26 @@ -package config +package server import ( "fmt" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" - "github.com/rancher/k3k/pkg/controller/util" + "github.com/rancher/k3k/pkg/controller" + "github.com/rancher/k3k/pkg/controller/cluster/agent" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -const ( - VirtualNodeMode = "virtual" -) - -// Server returns the secret for the server's config. Note that this doesn't set the ownerRef on the secret -// to tie it back to the cluster. -func Server(cluster *v1alpha1.Cluster, init bool, serviceIP string) (*v1.Secret, error) { - name := util.ServerConfigName(cluster) - if init { - name = util.ServerInitConfigName(cluster) - } - - cluster.Status.TLSSANs = append(cluster.Spec.TLSSANs, +func (s *Server) Config(init bool, serviceIP string) (*v1.Secret, error) { + name := configSecretName(s.cluster.Name, init) + s.cluster.Status.TLSSANs = append(s.cluster.Spec.TLSSANs, serviceIP, - util.ServerSvcName(cluster), - fmt.Sprintf("%s.%s", util.ServerSvcName(cluster), util.ClusterNamespace(cluster)), + ServiceName(s.cluster.Name), + fmt.Sprintf("%s.%s", ServiceName(s.cluster.Name), s.cluster.Namespace), ) - config := serverConfigData(serviceIP, cluster) + config := serverConfigData(serviceIP, s.cluster) if init { - config = initConfigData(cluster) + config = initConfigData(s.cluster) } return &v1.Secret{ TypeMeta: metav1.TypeMeta{ @@ -38,7 +29,7 @@ func Server(cluster *v1alpha1.Cluster, init bool, serviceIP string) (*v1.Secret, }, ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: util.ClusterNamespace(cluster), + Namespace: s.cluster.Namespace, }, Data: map[string][]byte{ "config.yaml": []byte(config), @@ -76,10 +67,17 @@ func serverOptions(cluster *v1alpha1.Cluster) string { opts = opts + "- " + addr + "\n" } } - if cluster.Spec.Mode != VirtualNodeMode { + if cluster.Spec.Mode != agent.VirtualNodeMode { opts = opts + "disable-agent: true\negress-selector-mode: disabled\n" } // TODO: Add extra args to the options return opts } + +func configSecretName(clusterName string, init bool) string { + if !init { + return controller.ObjectName(clusterName, nil, configName) + } + return controller.ObjectName(clusterName, nil, initConfigName) +} diff --git a/pkg/controller/cluster/server/ingress.go b/pkg/controller/cluster/server/ingress.go index 154cb5f..4c7b9fb 100644 --- a/pkg/controller/cluster/server/ingress.go +++ b/pkg/controller/cluster/server/ingress.go @@ -3,7 +3,7 @@ package server import ( "context" - "github.com/rancher/k3k/pkg/controller/util" + "github.com/rancher/k3k/pkg/controller" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -15,12 +15,13 @@ const ( nginxSSLPassthroughAnnotation = "nginx.ingress.kubernetes.io/ssl-passthrough" nginxBackendProtocolAnnotation = "nginx.ingress.kubernetes.io/backend-protocol" nginxSSLRedirectAnnotation = "nginx.ingress.kubernetes.io/ssl-redirect" - serverPort = 6443 - etcdPort = 2379 + + serverPort = 6443 + etcdPort = 2379 ) func (s *Server) Ingress(ctx context.Context, client client.Client) (*networkingv1.Ingress, error) { - addresses, err := util.Addresses(ctx, client) + addresses, err := controller.Addresses(ctx, client) if err != nil { return nil, err } @@ -31,8 +32,12 @@ func (s *Server) Ingress(ctx context.Context, client client.Client) (*networking APIVersion: "networking.k8s.io/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: s.cluster.Name + "-server-ingress", - Namespace: util.ClusterNamespace(s.cluster), + Name: controller.ObjectName(s.cluster.Name, &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{ + Kind: "Ingress", + }, + }), + Namespace: s.cluster.Namespace, }, Spec: networkingv1.IngressSpec{ IngressClassName: &s.cluster.Spec.Expose.Ingress.IngressClassName, @@ -59,7 +64,7 @@ func (s *Server) ingressRules(addresses []string) []networkingv1.IngressRule { PathType: &pathTypePrefix, Backend: networkingv1.IngressBackend{ Service: &networkingv1.IngressServiceBackend{ - Name: util.ServerSvcName(s.cluster), + Name: ServiceName(s.cluster.Name), Port: networkingv1.ServiceBackendPort{ Number: serverPort, }, diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index c8e382b..0e9c2ac 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -5,7 +5,7 @@ import ( "strings" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" - "github.com/rancher/k3k/pkg/controller/util" + "github.com/rancher/k3k/pkg/controller" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -17,12 +17,12 @@ import ( ) const ( - serverName = "k3k-" - k3kSystemNamespace = serverName + "system" - initServerName = serverName + "init-server" - initContainerName = serverName + "server-check" - initContainerImage = "alpine/curl" + k3kSystemNamespace = "k3k-system" + serverName = "server" + configName = "server-config" + initConfigName = "init-server-config" + ServerPort = 6443 EphermalNodesType = "ephermal" DynamicNodesType = "dynamic" ) @@ -40,7 +40,7 @@ func New(cluster *v1alpha1.Cluster, client client.Client) *Server { } } -func (s *Server) podSpec(ctx context.Context, image, name string, persistent bool, affinitySelector *metav1.LabelSelector) v1.PodSpec { +func (s *Server) podSpec(image, name string, persistent bool, affinitySelector *metav1.LabelSelector) v1.PodSpec { var limit v1.ResourceList if s.cluster.Spec.Limit != nil && s.cluster.Spec.Limit.ServerLimit != nil { limit = s.cluster.Spec.Limit.ServerLimit @@ -62,7 +62,7 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo Name: "initconfig", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: util.ServerInitConfigName(s.cluster), + SecretName: configSecretName(s.cluster.Name, true), Items: []v1.KeyToPath{ { Key: "config.yaml", @@ -76,7 +76,7 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo Name: "config", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ - SecretName: util.ServerConfigName(s.cluster), + SecretName: configSecretName(s.cluster.Name, false), Items: []v1.KeyToPath{ { Key: "config.yaml", @@ -220,18 +220,18 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo return podSpec } -func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster) (*apps.StatefulSet, error) { +func (s *Server) StatefulServer(ctx context.Context) (*apps.StatefulSet, error) { var ( replicas int32 pvClaims []v1.PersistentVolumeClaim persistent bool ) - image := util.K3SImage(cluster) - name := serverName + "server" + image := controller.K3SImage(s.cluster) + name := controller.ObjectName(s.cluster.Name, nil, serverName) - replicas = *cluster.Spec.Servers + replicas = *s.cluster.Spec.Servers - if cluster.Spec.Persistence != nil && cluster.Spec.Persistence.Type != EphermalNodesType { + if s.cluster.Spec.Persistence != nil && s.cluster.Spec.Persistence.Type != EphermalNodesType { persistent = true pvClaims = []v1.PersistentVolumeClaim{ { @@ -241,14 +241,14 @@ func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster) }, ObjectMeta: metav1.ObjectMeta{ Name: "varlibrancherk3s", - Namespace: util.ClusterNamespace(cluster), + Namespace: s.cluster.Namespace, }, Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, - StorageClassName: &cluster.Spec.Persistence.StorageClassName, + StorageClassName: &s.cluster.Spec.Persistence.StorageClassName, Resources: v1.VolumeResourceRequirements{ Requests: v1.ResourceList{ - "storage": resource.MustParse(cluster.Spec.Persistence.StorageRequestSize), + "storage": resource.MustParse(s.cluster.Spec.Persistence.StorageRequestSize), }, }, }, @@ -260,16 +260,16 @@ func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster) }, ObjectMeta: metav1.ObjectMeta{ Name: "varlibkubelet", - Namespace: util.ClusterNamespace(cluster), + Namespace: s.cluster.Namespace, }, Spec: v1.PersistentVolumeClaimSpec{ Resources: v1.VolumeResourceRequirements{ Requests: v1.ResourceList{ - "storage": resource.MustParse(cluster.Spec.Persistence.StorageRequestSize), + "storage": resource.MustParse(s.cluster.Spec.Persistence.StorageRequestSize), }, }, AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, - StorageClassName: &cluster.Spec.Persistence.StorageClassName, + StorageClassName: &s.cluster.Spec.Persistence.StorageClassName, }, }, } @@ -301,7 +301,7 @@ func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster) }, ObjectMeta: metav1.ObjectMeta{ Name: addons.Name, - Namespace: util.ClusterNamespace(s.cluster), + Namespace: s.cluster.Namespace, }, Data: make(map[string][]byte, len(addons.Data)), } @@ -335,12 +335,12 @@ func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster) selector := metav1.LabelSelector{ MatchLabels: map[string]string{ - "cluster": cluster.Name, + "cluster": s.cluster.Name, "role": "server", }, } - podSpec := s.podSpec(ctx, image, name, persistent, &selector) + podSpec := s.podSpec(image, name, persistent, &selector) podSpec.Volumes = append(podSpec.Volumes, volumes...) podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volumeMounts...) @@ -350,13 +350,13 @@ func (s *Server) StatefulServer(ctx context.Context, cluster *v1alpha1.Cluster) APIVersion: "apps/v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: cluster.Name + "-" + name, - Namespace: util.ClusterNamespace(cluster), + Name: name, + Namespace: s.cluster.Namespace, Labels: selector.MatchLabels, }, Spec: apps.StatefulSetSpec{ Replicas: &replicas, - ServiceName: cluster.Name + "-" + name + "-headless", + ServiceName: headlessServiceName(s.cluster.Name), Selector: &selector, VolumeClaimTemplates: pvClaims, Template: v1.PodTemplateSpec{ diff --git a/pkg/controller/cluster/server/service.go b/pkg/controller/cluster/server/service.go index 1ecbf6c..5fe253e 100644 --- a/pkg/controller/cluster/server/service.go +++ b/pkg/controller/cluster/server/service.go @@ -2,7 +2,7 @@ package server import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" - "github.com/rancher/k3k/pkg/controller/util" + "github.com/rancher/k3k/pkg/controller" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -23,8 +23,8 @@ func (s *Server) Service(cluster *v1alpha1.Cluster) *v1.Service { APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: util.ServerSvcName(cluster), - Namespace: util.ClusterNamespace(cluster), + Name: ServiceName(s.cluster.Name), + Namespace: cluster.Namespace, }, Spec: v1.ServiceSpec{ Type: serviceType, @@ -48,22 +48,21 @@ func (s *Server) Service(cluster *v1alpha1.Cluster) *v1.Service { } } -func (s *Server) StatefulServerService(cluster *v1alpha1.Cluster) *v1.Service { - name := serverName +func (s *Server) StatefulServerService() *v1.Service { return &v1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: cluster.Name + "-" + name + "headless", - Namespace: util.ClusterNamespace(cluster), + Name: headlessServiceName(s.cluster.Name), + Namespace: s.cluster.Namespace, }, Spec: v1.ServiceSpec{ Type: v1.ServiceTypeClusterIP, ClusterIP: v1.ClusterIPNone, Selector: map[string]string{ - "cluster": cluster.Name, + "cluster": s.cluster.Name, "role": "server", }, Ports: []v1.ServicePort{ @@ -81,3 +80,18 @@ func (s *Server) StatefulServerService(cluster *v1alpha1.Cluster) *v1.Service { }, } } + +func ServiceName(clusterName string) string { + return controller.ObjectName(clusterName, &v1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + }, + }) +} + +func headlessServiceName(clusterName string) string { + return controller.ObjectName(clusterName, &v1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + }}, "-headless") +} diff --git a/pkg/controller/clusterset/clusterset.go b/pkg/controller/clusterset/clusterset.go index a1827b0..1e62239 100644 --- a/pkg/controller/clusterset/clusterset.go +++ b/pkg/controller/clusterset/clusterset.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + k3kcontroller "github.com/rancher/k3k/pkg/controller" v1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -20,7 +21,6 @@ import ( const ( clusterSetController = "k3k-clusterset-controller" - networkPolicyName = "k3k-cluster-netpol" allTrafficCIDR = "0.0.0.0/0" maxConcurrentReconciles = 1 ) @@ -108,7 +108,7 @@ func netpol(ctx context.Context, clusterCIDR string, clusterSet *v1alpha1.Cluste } return &networkingv1.NetworkPolicy{ ObjectMeta: metav1.ObjectMeta{ - Name: networkPolicyName, + Name: k3kcontroller.ObjectName(clusterSet.Name, nil), Namespace: clusterSet.Namespace, }, TypeMeta: metav1.TypeMeta{ diff --git a/pkg/controller/clusterset/node.go b/pkg/controller/clusterset/node.go index 9cd23d7..fe2a7c9 100644 --- a/pkg/controller/clusterset/node.go +++ b/pkg/controller/clusterset/node.go @@ -4,7 +4,7 @@ import ( "context" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" - "github.com/rancher/k3k/pkg/controller/util" + k3kcontroller "github.com/rancher/k3k/pkg/controller" v1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/runtime" @@ -45,7 +45,7 @@ func AddNodeController(ctx context.Context, mgr manager.Manager) error { func (n *NodeReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { var clusterSetList v1alpha1.ClusterSetList if err := n.Client.List(ctx, &clusterSetList); err != nil { - return reconcile.Result{}, util.LogAndReturnErr("failed to list clusterSets", err) + return reconcile.Result{}, k3kcontroller.LogAndReturnErr("failed to list clusterSets", err) } if len(clusterSetList.Items) <= 0 { diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go new file mode 100644 index 0000000..8c5d537 --- /dev/null +++ b/pkg/controller/controller.go @@ -0,0 +1,106 @@ +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "strings" + "time" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + namePrefix = "k3k" + k3SImageName = "rancher/k3s" + AdminCommonName = "system:admin" +) + +// Backoff is the cluster creation duration backoff +var Backoff = wait.Backoff{ + Steps: 5, + Duration: 5 * time.Second, + Factor: 2, + Jitter: 0.1, +} + +func K3SImage(cluster *v1alpha1.Cluster) string { + return k3SImageName + ":" + cluster.Spec.Version +} + +func LogAndReturnErr(errString string, err error) error { + klog.Errorf("%s: %v", errString, err) + return err +} + +func nodeAddress(node *v1.Node) string { + var externalIP string + var internalIP string + + for _, ip := range node.Status.Addresses { + if ip.Type == "ExternalIP" && ip.Address != "" { + externalIP = ip.Address + break + } + if ip.Type == "InternalIP" && ip.Address != "" { + internalIP = ip.Address + } + } + if externalIP != "" { + return externalIP + } + + return internalIP +} + +// return all the nodes external addresses, if not found then return internal addresses +func Addresses(ctx context.Context, client ctrlruntimeclient.Client) ([]string, error) { + var nodeList v1.NodeList + if err := client.List(ctx, &nodeList); err != nil { + return nil, err + } + + var addresses []string + for _, node := range nodeList.Items { + addresses = append(addresses, nodeAddress(&node)) + } + + return addresses, nil +} + +// ObjectName will create a concatenated name based on the object's kind name that is being sent +// along with a prefix and the cluster name as well. +func ObjectName(clusterName string, object ctrlruntimeclient.Object, any ...string) string { + names := []string{namePrefix} + if clusterName != "" { + names = append(names, clusterName) + } + var objectKind string + if object != nil { + objectKind = strings.ToLower(object.GetObjectKind().GroupVersionKind().Kind) + names = append(names, objectKind) + } + return SafeConcatName(append(names, any...)...) +} + +// safeConcatName concatenates the given strings and ensures the returned name is under 64 characters +// by cutting the string off at 57 characters and setting the last 6 with an encoded version of the concatenated string. +func SafeConcatName(name ...string) string { + fullPath := strings.Join(name, "-") + if len(fullPath) < 64 { + return fullPath + } + digest := sha256.Sum256([]byte(fullPath)) + // since we cut the string in the middle, the last char may not be compatible with what is expected in k8s + // we are checking and if necessary removing the last char + c := fullPath[56] + if 'a' <= c && c <= 'z' || '0' <= c && c <= '9' { + return fullPath[0:57] + "-" + hex.EncodeToString(digest[0:])[0:5] + } + + return fullPath[0:56] + "-" + hex.EncodeToString(digest[0:])[0:6] +} diff --git a/pkg/controller/kubeconfig/kubeconfig.go b/pkg/controller/kubeconfig/kubeconfig.go index 2b8de59..0347cdd 100644 --- a/pkg/controller/kubeconfig/kubeconfig.go +++ b/pkg/controller/kubeconfig/kubeconfig.go @@ -10,8 +10,9 @@ import ( certutil "github.com/rancher/dynamiclistener/cert" "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" + "github.com/rancher/k3k/pkg/controller" + "github.com/rancher/k3k/pkg/controller/cluster/server" "github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap" - "github.com/rancher/k3k/pkg/controller/util" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" @@ -28,8 +29,8 @@ type KubeConfig struct { func (k *KubeConfig) Extract(ctx context.Context, client client.Client, cluster *v1alpha1.Cluster, hostServerIP string) ([]byte, error) { nn := types.NamespacedName{ - Name: cluster.Name + "-bootstrap", - Namespace: util.ClusterNamespace(cluster), + Name: controller.ObjectName(cluster.Name, nil, "bootstrap"), + Namespace: cluster.Namespace, } var bootstrapSecret v1.Secret @@ -57,8 +58,8 @@ func (k *KubeConfig) Extract(ctx context.Context, client client.Client, cluster } // get the server service to extract the right IP nn = types.NamespacedName{ - Name: util.ServerSvcName(cluster), - Namespace: util.ClusterNamespace(cluster), + Name: server.ServiceName(cluster.Name), + Namespace: cluster.Namespace, } var k3kService v1.Service @@ -66,7 +67,7 @@ func (k *KubeConfig) Extract(ctx context.Context, client client.Client, cluster return nil, err } - url := fmt.Sprintf("https://%s:%d", k3kService.Spec.ClusterIP, util.ServerPort) + url := fmt.Sprintf("https://%s:%d", k3kService.Spec.ClusterIP, server.ServerPort) if k3kService.Spec.Type == v1.ServiceTypeNodePort { nodePort := k3kService.Spec.Ports[0].NodePort url = fmt.Sprintf("https://%s:%d", hostServerIP, nodePort) diff --git a/pkg/controller/util/util.go b/pkg/controller/util/util.go deleted file mode 100644 index 4c86651..0000000 --- a/pkg/controller/util/util.go +++ /dev/null @@ -1,87 +0,0 @@ -package util - -import ( - "context" - "fmt" - - "github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1" - v1 "k8s.io/api/core/v1" - "k8s.io/klog" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - namespacePrefix = "k3k-" - k3SImageName = "rancher/k3s" - - AdminCommonName = "system:admin" - ServerPort = 6443 -) - -const ( - K3kSystemNamespace = namespacePrefix + "system" -) - -func ClusterNamespace(cluster *v1alpha1.Cluster) string { - return cluster.Namespace -} - -func ServerSvcName(cluster *v1alpha1.Cluster) string { - return fmt.Sprintf("k3k-%s-service", cluster.Name) -} - -func ServerConfigName(cluster *v1alpha1.Cluster) string { - return fmt.Sprintf("k3k-%s-server-config", cluster.Name) -} - -func ServerInitConfigName(cluster *v1alpha1.Cluster) string { - return fmt.Sprintf("k3k-init-%s-server-config", cluster.Name) -} - -func AgentConfigName(cluster *v1alpha1.Cluster) string { - return fmt.Sprintf("k3k-%s-agent-config", cluster.Name) -} - -func K3SImage(cluster *v1alpha1.Cluster) string { - return k3SImageName + ":" + cluster.Spec.Version -} - -func LogAndReturnErr(errString string, err error) error { - klog.Errorf("%s: %v", errString, err) - return err -} - -func nodeAddress(node *v1.Node) string { - var externalIP string - var internalIP string - - for _, ip := range node.Status.Addresses { - if ip.Type == "ExternalIP" && ip.Address != "" { - externalIP = ip.Address - break - } - if ip.Type == "InternalIP" && ip.Address != "" { - internalIP = ip.Address - } - } - if externalIP != "" { - return externalIP - } - - return internalIP -} - -// return all the nodes external addresses, if not found then return internal addresses -func Addresses(ctx context.Context, client client.Client) ([]string, error) { - var nodeList v1.NodeList - if err := client.List(ctx, &nodeList); err != nil { - return nil, err - } - - var addresses []string - for _, node := range nodeList.Items { - addresses = append(addresses, nodeAddress(&node)) - } - - return addresses, nil -}