Move supervisor API request handlers into dedicated package and add unit tests

pkg/agent/config/config.go

@@ -2,9 +2,11 @@ package config
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	cryptorand "crypto/rand"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/hex"
 	"encoding/pem"
 	"fmt"
@@ -32,6 +34,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
 	"github.com/pkg/errors"
+	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/rancher/wharfie/pkg/registries"
 	"github.com/rancher/wrangler/v3/pkg/slice"
 	"github.com/sirupsen/logrus"
@@ -133,9 +136,9 @@ func Request(path string, info *clientaccess.Info, requester HTTPRequester) ([]b
 	return requester(u.String(), clientaccess.GetHTTPClient(info.CACerts, info.CertFile, info.KeyFile), info.Username, info.Password, info.Token())
 }
 
-func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string) HTTPRequester {
+func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string, csr []byte) HTTPRequester {
 	return func(u string, client *http.Client, username, password, token string) ([]byte, error) {
-		req, err := http.NewRequest(http.MethodGet, u, nil)
+		req, err := http.NewRequest(http.MethodPost, u, bytes.NewReader(csr))
 		if err != nil {
 			return nil, err
 		}
@@ -238,51 +241,93 @@ func upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile string)
 	}
 }
 
-func getServingCert(nodeName string, nodeIPs []net.IP, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) {
-	servingCert, err := Request("/v1-"+version.Program+"/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile))
+// getKubeletServingCert fills the kubelet server certificate with content returned
+// from the server.  We attempt to POST a CSR to the server, in hopes that it will
+// sign the cert using our locally generated key. If the server does not support CSR
+// signing, the key generated by the server is used instead.
+func getKubeletServingCert(nodeName string, nodeIPs []net.IP, certFile, keyFile, nodePasswordFile string, info *clientaccess.Info) error {
+	csr, err := getCSRBytes(keyFile)
 	if err != nil {
-		return nil, err
+		return errors.Wrapf(err, "failed to create certificate request %s", certFile)
 	}
 
-	servingCert, servingKey := splitCertKeyPEM(servingCert)
-
-	if err := os.WriteFile(servingCertFile, servingCert, 0600); err != nil {
-		return nil, errors.Wrapf(err, "failed to write node cert")
+	basename := filepath.Base(certFile)
+	body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile, csr))
+	if err != nil {
+		return err
 	}
 
-	if err := os.WriteFile(servingKeyFile, servingKey, 0600); err != nil {
-		return nil, errors.Wrapf(err, "failed to write node key")
+	// Always split the response, as down-level servers may send back a cert+key
+	// instead of signing a new cert with our key.  If the response includes a key it
+	// must be used instead of the one we signed the CSR with.
+	certBytes, keyBytes := splitCertKeyPEM(body)
+	if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", certFile)
 	}
-
-	cert, err := tls.X509KeyPair(servingCert, servingKey)
-	if err != nil {
-		return nil, err
+	if len(keyBytes) > 0 {
+		if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
+			return errors.Wrapf(err, "failed to write key %s", keyFile)
+		}
 	}
-	return &cert, nil
+	return nil
 }
 
-func getHostFile(filename, keyFile string, info *clientaccess.Info) error {
+// getHostFile fills a file with content returned from the server.
+func getHostFile(filename string, info *clientaccess.Info) error {
 	basename := filepath.Base(filename)
 	fileBytes, err := info.Get("/v1-" + version.Program + "/" + basename)
 	if err != nil {
 		return err
 	}
-	if keyFile == "" {
-		if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write cert %s", filename)
-		}
-	} else {
-		fileBytes, keyBytes := splitCertKeyPEM(fileBytes)
-		if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write cert %s", filename)
-		}
+	if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", filename)
+	}
+	return nil
+}
+
+// getClientCert fills a client certificate with content returned from the server.
+// We attempt to POST a CSR to the server, in hopes that it will sign the cert using
+// our locally generated key. If the server does not support CSR signing, the key
+// generated by the server is used instead.
+func getClientCert(certFile, keyFile string, info *clientaccess.Info) error {
+	csr, err := getCSRBytes(keyFile)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create certificate request %s", certFile)
+	}
+
+	basename := filepath.Base(certFile)
+	fileBytes, err := info.Post("/v1-"+version.Program+"/"+basename, csr)
+	if err != nil {
+		return err
+	}
+
+	// Always split the response, as down-level servers may send back a cert+key
+	// instead of signing a new cert with our key.  If the response includes a key it
+	// must be used instead of the one we signed the CSR with.
+	certBytes, keyBytes := splitCertKeyPEM(fileBytes)
+	if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", certFile)
+	}
+	if len(keyBytes) > 0 {
 		if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write key %s", filename)
+			return errors.Wrapf(err, "failed to write key %s", keyFile)
 		}
 	}
 	return nil
 }
 
+func getCSRBytes(keyFile string) ([]byte, error) {
+	keyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile, false)
+	if err != nil {
+		return nil, err
+	}
+	key, err := certutil.ParsePrivateKeyPEM(keyBytes)
+	if err != nil {
+		return nil, err
+	}
+	return x509.CreateCertificateRequest(cryptorand.Reader, &x509.CertificateRequest{}, key)
+}
+
 func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) {
 	for {
 		b, rest := pem.Decode(bytes)
@@ -301,19 +346,33 @@ func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) {
 	return
 }
 
-func getNodeNamedHostFile(filename, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error {
-	basename := filepath.Base(filename)
-	fileBytes, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile))
+// getKubeletClientCert fills the kubelet client certificate with content returned
+// from the server.  We attempt to POST a CSR to the server, in hopes that it will
+// sign the cert using our locally generated key. If the server does not support CSR
+// signing, the key generated by the server is used instead.
+func getKubeletClientCert(certFile, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error {
+	csr, err := getCSRBytes(keyFile)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create certificate request %s", certFile)
+	}
+
+	basename := filepath.Base(certFile)
+	body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile, csr))
 	if err != nil {
 		return err
 	}
-	fileBytes, keyBytes := splitCertKeyPEM(fileBytes)
 
-	if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
-		return errors.Wrapf(err, "failed to write cert %s", filename)
+	// Always split the response, as down-level servers may send back a cert+key
+	// instead of signing a new cert with our key.  If the response includes a key it
+	// must be used instead of the one we signed the CSR with.
+	certBytes, keyBytes := splitCertKeyPEM(body)
+	if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", certFile)
 	}
-	if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
-		return errors.Wrapf(err, "failed to write key %s", filename)
+	if len(keyBytes) > 0 {
+		if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
+			return errors.Wrapf(err, "failed to write key %s", keyFile)
+		}
 	}
 	return nil
 }
@@ -399,12 +458,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	}
 
 	clientCAFile := filepath.Join(envInfo.DataDir, "agent", "client-ca.crt")
-	if err := getHostFile(clientCAFile, "", info); err != nil {
+	if err := getHostFile(clientCAFile, info); err != nil {
 		return nil, err
 	}
 
 	serverCAFile := filepath.Join(envInfo.DataDir, "agent", "server-ca.crt")
-	if err := getHostFile(serverCAFile, "", info); err != nil {
+	if err := getHostFile(serverCAFile, info); err != nil {
 		return nil, err
 	}
 
@@ -498,14 +557,13 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	// that the cert will not be valid for, as they are not present in the list collected here.
 	nodeExternalAndInternalIPs := append(nodeIPs, nodeExternalIPs...)
 
-	// Ask the server to generate a kubelet server cert+key. These files are unique to this node.
-	servingCert, err := getServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info)
-	if err != nil {
+	// Ask the server to sign our kubelet server cert.
+	if err := getKubeletServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil {
 		return nil, errors.Wrap(err, servingKubeletCert)
 	}
 
-	// Ask the server to genrate a kubelet client cert+key. These files are unique to this node.
-	if err := getNodeNamedHostFile(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
+	// Ask the server to sign our kubelet client cert.
+	if err := getKubeletClientCert(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
 		return nil, errors.Wrap(err, clientKubeletCert)
 	}
 
@@ -518,8 +576,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	clientKubeProxyCert := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.crt")
 	clientKubeProxyKey := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.key")
 
-	// Ask the server to send us its kube-proxy client cert+key. These files are not unique to this node.
-	if err := getHostFile(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
+	// Ask the server to sign our kube-proxy client cert.
+	if err := getClientCert(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
 		return nil, errors.Wrap(err, clientKubeProxyCert)
 	}
 
@@ -532,8 +590,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	clientK3sControllerCert := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.crt")
 	clientK3sControllerKey := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.key")
 
-	// Ask the server to send us its agent controller client cert+key. These files are not unique to this node.
-	if err := getHostFile(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
+	// Ask the server to sign our agent controller client cert.
+	if err := getClientCert(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
 		return nil, errors.Wrap(err, clientK3sControllerCert)
 	}
 
@@ -625,7 +683,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	applyCRIDockerdAddress(nodeConfig)
 	applyContainerdQoSClassConfigFileIfPresent(envInfo, &nodeConfig.Containerd)
 	nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "config.toml.tmpl")
-	nodeConfig.Certificate = servingCert
 
 	if envInfo.BindAddress != "" {
 		nodeConfig.AgentConfig.ListenAddress = envInfo.BindAddress

pkg/agent/flannel/setup_test.go

@@ -62,7 +62,7 @@ func Test_createFlannelConf(t *testing.T) {
 		var agent = config.Agent{}
 		agent.ClusterCIDR = stringToCIDR(tt.args)[0]
 		agent.ClusterCIDRs = stringToCIDR(tt.args)
-		var nodeConfig = &config.Node{Docker: false, ContainerRuntimeEndpoint: "", SELinux: false, FlannelBackend: "vxlan", FlannelConfFile: "test_file", FlannelConfOverride: false, FlannelIface: nil, Containerd: containerd, Images: "", AgentConfig: agent, Token: "", Certificate: nil, ServerHTTPSPort: 0}
+		var nodeConfig = &config.Node{Docker: false, ContainerRuntimeEndpoint: "", SELinux: false, FlannelBackend: "vxlan", FlannelConfFile: "test_file", FlannelConfOverride: false, FlannelIface: nil, Containerd: containerd, Images: "", AgentConfig: agent, Token: "", ServerHTTPSPort: 0}
 
 		t.Run(tt.name, func(t *testing.T) {
 			if err := createFlannelConf(nodeConfig); (err != nil) != tt.wantErr {

pkg/cli/secretsencrypt/secrets_encrypt.go

@@ -15,6 +15,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/proctitle"
 	"github.com/k3s-io/k3s/pkg/secretsencrypt"
 	"github.com/k3s-io/k3s/pkg/server"
+	"github.com/k3s-io/k3s/pkg/server/handlers"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
@@ -54,7 +55,7 @@ func Enable(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(server.EncryptionRequest{Enable: ptr.To(true)})
+	b, err := json.Marshal(handlers.EncryptionRequest{Enable: ptr.To(true)})
 	if err != nil {
 		return err
 	}
@@ -73,7 +74,7 @@ func Disable(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(server.EncryptionRequest{Enable: ptr.To(false)})
+	b, err := json.Marshal(handlers.EncryptionRequest{Enable: ptr.To(false)})
 	if err != nil {
 		return err
 	}
@@ -96,7 +97,7 @@ func Status(app *cli.Context) error {
 	if err != nil {
 		return wrapServerError(err)
 	}
-	status := server.EncryptionState{}
+	status := handlers.EncryptionState{}
 	if err := json.Unmarshal(data, &status); err != nil {
 		return err
 	}
@@ -153,7 +154,7 @@ func Prepare(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(server.EncryptionRequest{
+	b, err := json.Marshal(handlers.EncryptionRequest{
 		Stage: ptr.To(secretsencrypt.EncryptionPrepare),
 		Force: cmds.ServerConfig.EncryptForce,
 	})
@@ -175,7 +176,7 @@ func Rotate(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(server.EncryptionRequest{
+	b, err := json.Marshal(handlers.EncryptionRequest{
 		Stage: ptr.To(secretsencrypt.EncryptionRotate),
 		Force: cmds.ServerConfig.EncryptForce,
 	})
@@ -197,7 +198,7 @@ func Reencrypt(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(server.EncryptionRequest{
+	b, err := json.Marshal(handlers.EncryptionRequest{
 		Stage: ptr.To(secretsencrypt.EncryptionReencryptActive),
 		Force: cmds.ServerConfig.EncryptForce,
 		Skip:  cmds.ServerConfig.EncryptSkip,
@@ -220,7 +221,7 @@ func RotateKeys(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(server.EncryptionRequest{
+	b, err := json.Marshal(handlers.EncryptionRequest{
 		Stage: ptr.To(secretsencrypt.EncryptionRotateKeys),
 	})
 	if err != nil {

pkg/cli/token/token.go

@@ -16,6 +16,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/kubeadm"
 	"github.com/k3s-io/k3s/pkg/proctitle"
 	"github.com/k3s-io/k3s/pkg/server"
+	"github.com/k3s-io/k3s/pkg/server/handlers"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
@@ -153,7 +154,7 @@ func Rotate(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(server.TokenRotateRequest{
+	b, err := json.Marshal(handlers.TokenRotateRequest{
 		NewToken: ptr.To(cmds.TokenConfig.NewToken),
 	})
 	if err != nil {

pkg/cluster/https.go

@@ -141,9 +141,13 @@ func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error {
 func tlsStorage(ctx context.Context, dataDir string, runtime *config.ControlRuntime) dynamiclistener.TLSStorage {
 	fileStorage := file.New(filepath.Join(dataDir, "tls/dynamic-cert.json"))
 	cache := memory.NewBacked(fileStorage)
-	return kubernetes.New(ctx, func() *core.Factory {
-		return runtime.Core
-	}, metav1.NamespaceSystem, version.Program+"-serving", cache)
+	coreGetter := func() *core.Factory {
+		if coreFactory, ok := runtime.Core.(*core.Factory); ok {
+			return coreFactory
+		}
+		return nil
+	}
+	return kubernetes.New(ctx, coreGetter, metav1.NamespaceSystem, version.Program+"-serving", cache)
 }
 
 // wrapHandler wraps the dynamiclistener request handler, adding a User-Agent value to