Update kubelet config (#13)

* update /etc/kubernetes/kubelet.conf to use symlink /var/lib/kubelet/pki/kubelet-client-current.pem
* support enable/disable kubelet client and server certificate

Signed-off-by: JenTing Hsiao <[email protected]>

.github/workflows/ci.yml

@@ -10,10 +10,10 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go ^1.13
+      - name: Set up Go ^1.14
         uses: actions/setup-go@v2
         with:
-          go-version: ^1.13
+          go-version: ^1.14
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14.4-stretch as build
+FROM golang:1.14-stretch as build
 WORKDIR /src
 
 ARG VERSION=latest

README.md

@@ -8,19 +8,29 @@ Kucero (KUbernetes CErtificate ROtation) is a Kubernetes daemonset that
 performs _automatic_ Kubernetes control plane certificate rotation.
 
 Kucero takes care both:
-- kubeadm-managed certificates and kubeconfigs: kucero periodically watches the kubeadm generated certificates and kubeconfigs on host system, and renews certificates/kubeconfigs when the certificates/kubeconfigs residual time is below than user configured time period.
-- kubelet server CSR: kucero controller watches kubelet server CSR, and then auto signs and approves kubelet server certificates with user-specified CA cert/key pair.
+- kubeadm certificates and kubeconfigs: kucero periodically watches the kubeadm generated certificates and kubeconfigs on host system, and renews certificates/kubeconfigs when the certificates/kubeconfigs residual time is below than user configured time period.
+- kubelet certificates:
+  - kubelet.conf: kucero helps on auto-update the `/etc/kubernetes/kubelet.conf` from embedded base64 encoded client cert/key to using the local file `/var/lib/kubelet/kubelet-client-current.pem` (this is a bug if you bootstrap a cluster with kubeadm version < 1.17).
+  - client certificate: kucero helps on configuring `rotateCertificates: true` or `rotateCertificates: false` in `/var/lib/kubelet/config.yaml` which controls to auto rotates the kubelet client certificate or not. When configures `rotateCertificates: true`, the kubelet sends out the client CSR at approximately 70%-90% of the total lifetime of the certificate, then the kube-controler-manager watches kubelet client CSR, and then auto signs and approves kubelet client certificates with Kubernetes cluster CA cert/key pair.
+  - server certificate: kucero helps on configuring `serverTLSBootstrap: true` or `serverTLSBootstrap: false` in `/var/lib/kubelet/config.yaml` which controls to auto rotates the kubelet server certificate or not. When configures `serverTLSBootstrap: true`, the kubelet sends out the server CSR at approximately 70%-90% of the total lifetime of the certificate, then the kucero controller watches kubelet server CSR, and then auto signs and approves kubelet server certificates with user-specified CA cert/key pair.
 
 ## Kubelet Configuration
 
-To enable kubelet server TLS bootstrapping, you need to set `serverTLSBootstrap: true` in kubelet configuration file `/var/lib/kubelet/config.yaml`. This will enable kubelet to generates kubelet server CSR.
+By default, kucero enables kubelet client `rotateCertificates: true` and server certificates `serverTLSBootstrap: true` auto rotation, you could disable it by passing flags to kucero:
+- `--enable-kubelet-client-cert-rotation=false`
+- `--enable-kubelet-server-cert-rotation=false`
 
 ## Build Requirements
 
 - Golang >= 1.13
 - Docker
 - Kustomize
 
+## Container Requirement Package
+
+- /usr/bin/kubectl
+- /usr/bin/nsenter
+
 ## Kubeadm Compatibility
 
 - kubeadm >= 1.15.0
@@ -52,7 +62,11 @@ Flags:
       --renew-before duration       rotates certificate before expiry is below (default 720h0m0s)
 ```
 
-##
+## Uninstallation
+
+```
+make destroy-manifest
+```
 
 ## Demo
 

cmd/kucero/main.go

@@ -38,7 +38,7 @@ import (
 
 	"github.com/jenting/kucero/controllers"
 	"github.com/jenting/kucero/pkg/host"
-	"github.com/jenting/kucero/pkg/pki/cert"
+	"github.com/jenting/kucero/pkg/pki/node"
 	"github.com/jenting/kucero/pkg/pki/signer"
 )
 
@@ -53,6 +53,8 @@ var (
 	metricsAddr                                 string
 	leaderElectionID                            string
 	caCertPath, caKeyPath                       string
+	enableKubeletClientCertRotation             bool
+	enableKubeletServerCertRotation             bool
 
 	scheme = runtime.NewScheme()
 )
@@ -102,6 +104,12 @@ func main() {
 	rootCmd.PersistentFlags().DurationVar(&duration, "duration", time.Hour*24*365,
 		"Kubelet certificate duration")
 
+	// kubelet configuration
+	rootCmd.PersistentFlags().BoolVar(&enableKubeletClientCertRotation, "enable-kubelet-client-cert-rotation", true,
+		"Enable kubelet client cert rotation")
+	rootCmd.PersistentFlags().BoolVar(&enableKubeletServerCertRotation, "enable-kubelet-server-cert-rotation", true,
+		"Enable kubelet server cert rotation")
+
 	if err := rootCmd.Execute(); err != nil {
 		logrus.Error(err)
 	}
@@ -120,26 +128,7 @@ func root(cmd *cobra.Command, args []string) {
 	extra := rand.Intn(int(pollingPeriod.Seconds()))
 	pollingPeriod = pollingPeriod + time.Duration(extra)*time.Second
 
-	logrus.Infof("Node Name: %s", nodeName)
-	logrus.Infof("Lock Annotation: %s/%s:%s", dsNamespace, dsName, lockAnnotation)
-	logrus.Infof("Shifted Certificate Check Polling Period %v", pollingPeriod)
-	logrus.Infof("Rotates Certificate If Expiry Time Less Than %v", expiryTimeToRotate)
-	if enableKubeletCSRController {
-		logrus.Infof("Leader election ID: %s", leaderElectionID)
-		logrus.Infof("CA cert: %s", caCertPath)
-		logrus.Infof("CA key: %s", caKeyPath)
-	}
-
-	rotateCertificateWhenNeeded(nodeName)
-}
-
-// nodeMeta is used to remember information across nodes
-// whom is doing certificate rotation
-type nodeMeta struct {
-	Unschedulable bool `json:"unschedulable"`
-}
-
-func rotateCertificateWhenNeeded(nodeName string) {
+	// check it's a control plane node or worker node
 	config, err := clientcmd.BuildConfigFromFlags(apiServerHost, kubeconfig)
 	if err != nil {
 		logrus.Fatal(err)
@@ -150,27 +139,48 @@ func rotateCertificateWhenNeeded(nodeName string) {
 		logrus.Fatal(err)
 	}
 
-	node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
+	corev1Node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
 	if err != nil {
 		logrus.Fatal(err)
 	}
 
 	isControlPlaneNode := true
-	_, exist := node.GetLabels()["node-role.kubernetes.io/master"]
+	_, exist := corev1Node.GetLabels()["node-role.kubernetes.io/master"]
 	if !exist {
 		isControlPlaneNode = false
-		logrus.Fatal("Kucero supports running on control plane node only")
 	}
 
-	certNode := cert.NewNode(isControlPlaneNode, nodeName, expiryTimeToRotate)
+	logrus.Infof("Node Name: %s", nodeName)
+	logrus.Infof("Lock Annotation: %s/%s:%s", dsNamespace, dsName, lockAnnotation)
+	logrus.Infof("Shifted Certificate Check Polling Period %v", pollingPeriod)
+	logrus.Infof("Rotates Certificate If Expiry Time Less Than %v", expiryTimeToRotate)
+	logrus.Infof("Kubelet client cert rotation enabled: %t", enableKubeletClientCertRotation)
+	logrus.Infof("Kubelet server cert rotation enabled: %t", enableKubeletServerCertRotation)
+	if enableKubeletCSRController && isControlPlaneNode {
+		logrus.Infof("Kubelet CSR controller leader election ID: %s", leaderElectionID)
+		logrus.Infof("Kubelet CSR controller CA cert: %s", caCertPath)
+		logrus.Infof("Kubelet CSR controller CA key: %s", caKeyPath)
+	}
+
+	rotateCertificateWhenNeeded(nodeName, isControlPlaneNode, client)
+}
+
+// nodeMeta is used to remember information across nodes
+// whom is doing certificate rotation
+type nodeMeta struct {
+	Unschedulable bool `json:"unschedulable"`
+}
+
+func rotateCertificateWhenNeeded(nodeName string, isControlPlaneNode bool, client *kubernetes.Clientset) {
+	certNode := node.New(isControlPlaneNode, nodeName, expiryTimeToRotate, enableKubeletClientCertRotation, enableKubeletServerCertRotation)
 
 	lock := daemonsetlock.New(client, nodeName, dsNamespace, dsName, lockAnnotation)
 	nodeMeta := nodeMeta{}
 	if holding(lock, &nodeMeta) {
 		release(lock)
 	}
 
-	if enableKubeletCSRController {
+	if enableKubeletCSRController && isControlPlaneNode {
 		go func() {
 			mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 				Scheme:                  scheme,
@@ -219,6 +229,12 @@ func rotateCertificateWhenNeeded(nodeName string) {
 		case <-ch:
 			logrus.Info("Check certificate expiration")
 
+			// check the configuration needs to be update
+			configsToBeUpdate, err := certNode.CheckConfig()
+			if err != nil {
+				logrus.Error(err)
+			}
+
 			// check the certificate needs expiration
 			expiryCerts, err := certNode.CheckExpiration()
 			if err != nil {
@@ -229,22 +245,34 @@ func rotateCertificateWhenNeeded(nodeName string) {
 			// and the lock can be acquired.
 			// if the lock cannot be acquired, it will wait `pollingPeriod` time
 			// and try to acquire the lock again.
-			if len(expiryCerts) > 0 && acquire(lock, &nodeMeta) {
-				logrus.Infof("The expiry certificiates are %v", expiryCerts)
-
+			if (len(configsToBeUpdate) > 0 || len(expiryCerts) > 0) && acquire(lock, &nodeMeta) {
 				if !nodeMeta.Unschedulable {
-					host.Cordon(nodeName)
-					host.Drain(nodeName)
+					_ = host.Cordon(nodeName)
+					_ = host.Drain(nodeName)
 				}
 
-				logrus.Info("Waiting for certificate rotation")
-				if err := certNode.Rotate(expiryCerts); err != nil {
-					logrus.Error(err)
+				if len(configsToBeUpdate) > 0 {
+					logrus.Infof("The configuration need to be updates are %v", configsToBeUpdate)
+
+					logrus.Info("Waiting for configuration to be update")
+					if err := certNode.UpdateConfig(configsToBeUpdate); err != nil {
+						logrus.Error(err)
+					}
+					logrus.Info("Update configuration done")
+				}
+
+				if len(expiryCerts) > 0 {
+					logrus.Infof("The expiry certificiates are %v", expiryCerts)
+
+					logrus.Info("Waiting for certificate rotation")
+					if err := certNode.Rotate(expiryCerts); err != nil {
+						logrus.Error(err)
+					}
+					logrus.Info("Certificate rotation done")
 				}
-				logrus.Info("Certificate rotation done")
 
 				if !nodeMeta.Unschedulable {
-					host.Uncordon(nodeName)
+					_ = host.Uncordon(nodeName)
 				}
 
 				release(lock)

go.mod

@@ -19,5 +19,5 @@ require (
 	k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 // indirect
 	k8s.io/utils v0.0.0-20200414100711-2df71ebbae66 // indirect
 	sigs.k8s.io/controller-runtime v0.5.0
-	sigs.k8s.io/yaml v1.2.0 // indirect
+	sigs.k8s.io/yaml v1.2.0
 )

manifest/daemonset.yaml

@@ -20,18 +20,24 @@ spec:
         - key: node-role.kubernetes.io/master
           operator: Exists
           effect: NoSchedule
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
       hostPID: true # Facilitate entering the host mount namespace via init
       restartPolicy: Always
       volumes:
+        - name: kubelet-conf
+          hostPath:
+            path: /etc/kubernetes/kubelet.conf
+            type: File
         - name: ca-crt
           hostPath:
             path: /etc/kubernetes/pki/ca.crt
             type: File
         - name: ca-key
           hostPath:
             path: /etc/kubernetes/pki/ca.key
+            type: FileOrCreate
+        - name: kubelet-config-yaml
+          hostPath:
+            path: /var/lib/kubelet/config.yaml
             type: File
       containers:
         - name: kucero
@@ -49,9 +55,13 @@ spec:
           command:
             - /usr/bin/kucero
           volumeMounts:
+            - mountPath: /etc/kubernetes/kubelet.conf
+              name: kubelet-conf
             - mountPath: /etc/kubernetes/pki/ca.crt
               name: ca-crt
               readOnly: true
             - mountPath: /etc/kubernetes/pki/ca.key
               name: ca-key
               readOnly: true
+            - mountPath: /var/lib/kubelet/config.yaml
+              name: kubelet-config-yaml

manifest/privileged.yaml

@@ -17,8 +17,10 @@ spec:
     - secret
     - hostPath
   allowedHostPaths:
+    - pathPrefix: /etc/kubernetes
     - pathPrefix: /etc/kubernetes/pki
       readOnly: true
+    - pathPrefix: /var/lib/kubelet
     - pathPrefix: /var/lib/kubelet/pki
       readOnly: true
 ---

pkg/pki/cert/node.go → pkg/pki/cert/certificate.go

@@ -16,18 +16,12 @@ limitations under the License.
 
 package cert
 
-import (
-	"time"
-
-	"github.com/jenting/kucero/pkg/pki/cert/node"
-)
-
-// NewNode checks it is control plane node or worker node
-// then returns the corresponding node certificate interface
-func NewNode(isControlPlaneNode bool, nodeName string, expiryTimeToRotate time.Duration) node.Certificate {
-	if !isControlPlaneNode {
-		return nil
-	}
-
-	return node.NewControlPlaneNode(nodeName, expiryTimeToRotate)
+type Certificate interface {
+	// CheckExpiration checks node certificate
+	// returns the certificates which are going to expires
+	CheckExpiration() ([]string, error)
+
+	// Rotate rotates the node certificates
+	// which are going to expires
+	Rotate(expiryCertificates []string) error
 }