Merge remote-tracking branch 'upstream/master'

.drone.yml

@@ -617,33 +617,24 @@ steps:
   - mkdir -p dist/artifacts
   - cp /tmp/artifacts/* dist/artifacts/
   - docker stop registry && docker rm registry
-  # Cleanup VMs running, happens if a previous test panics
-  # Cleanup inactive domains, happens if previous test is canceled
-  - |
-    VMS=$(virsh list --name | grep '_server-\|_agent-' || true)
-    if [ -n "$VMS" ]; then
-      for vm in $VMS
-      do
-        virsh destroy $vm
-        virsh undefine $vm --remove-all-storage
-      done
-    fi 
-    VMS=$(virsh list --name --inactive | grep '_server-\|_agent-' || true)
-    if [ -n "$VMS" ]; then
-      for vm in $VMS
-      do
-        virsh undefine $vm
-      done
-    fi 
+  # Cleanup VMs that are older than 2h. Happens if a previous test panics or is canceled
+  - tests/e2e/scripts/cleanup_vms.sh
   - docker run -d -p 5000:5000 -e REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io --name registry registry:2
-  - cd tests/e2e/validatecluster
-  - vagrant destroy -f
-  - go test -v -timeout=45m ./validatecluster_test.go -ci -local
-  - cp ./coverage.out /tmp/artifacts/validate-coverage.out
-  - cd ../secretsencryption
-  - vagrant destroy -f
-  - go test -v -timeout=30m ./secretsencryption_test.go -ci -local
-  - cp ./coverage.out /tmp/artifacts/se-coverage.out
+  - |
+    cd tests/e2e/validatecluster
+    vagrant destroy -f
+    go test -v -timeout=45m ./validatecluster_test.go -ci -local
+    cp ./coverage.out /tmp/artifacts/validate-coverage.out
+  - |
+    cd ../secretsencryption
+    vagrant destroy -f
+    go test -v -timeout=30m ./secretsencryption_test.go -ci -local
+    cp ./coverage.out /tmp/artifacts/se-coverage.out
+  - |
+    cd ../splitserver
+    vagrant destroy -f
+    go test -v -timeout=30m ./splitserver_test.go -ci -local
+    cp ./coverage.out /tmp/artifacts/split-coverage.out
   - |
     if [ "$DRONE_BUILD_EVENT" = "pull_request" ]; then
       cd ../upgradecluster
@@ -669,6 +660,7 @@ steps:
     files:
       - /tmp/artifacts/validate-coverage.out
       - /tmp/artifacts/se-coverage.out
+      - /tmp/artifacts/split-coverage.out
       - /tmp/artifacts/upgrade-coverage.out
     flags: 
       - e2etests

channel.yaml

@@ -1,7 +1,7 @@
 # Example channels config
 channels:
 - name: stable
-  latest: v1.29.3+k3s1
+  latest: v1.29.4+k3s1
 - name: latest
   latestRegexp: .*
   excludeRegexp: (^[^+]+-|v1\.25\.5\+k3s1|v1\.26\.0\+k3s1)

pkg/cli/etcdsnapshot/etcd_snapshot.go

@@ -2,6 +2,7 @@ package etcdsnapshot
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -26,6 +27,8 @@ import (
 	"k8s.io/cli-runtime/pkg/printers"
 )
 
+var timeout = 2 * time.Minute
+
 // commandSetup setups up common things needed
 // for each etcd command.
 func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *clientaccess.Info, error) {
@@ -58,6 +61,8 @@ func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *c
 		sr.S3.SecretKey = cfg.EtcdS3SecretKey
 		sr.S3.SkipSSLVerify = cfg.EtcdS3SkipSSLVerify
 		sr.S3.Timeout = metav1.Duration{Duration: cfg.EtcdS3Timeout}
+		// extend request timeout to allow the S3 operation to complete
+		timeout += cfg.EtcdS3Timeout
 	}
 
 	dataDir, err := server.ResolveDataDir(cfg.DataDir)
@@ -78,6 +83,11 @@ func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *c
 }
 
 func wrapServerError(err error) error {
+	if errors.Is(err, context.DeadlineExceeded) {
+		// if the request timed out the server log likely won't contain anything useful,
+		// since the operation may have actualy succeeded despite the client timing out the request.
+		return err
+	}
 	return errors.Wrap(err, "see server log for details")
 }
 
@@ -110,7 +120,7 @@ func save(app *cli.Context, cfg *cmds.Server) error {
 	if err != nil {
 		return err
 	}
-	r, err := info.Post("/db/snapshot", b)
+	r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout))
 	if err != nil {
 		return wrapServerError(err)
 	}
@@ -151,7 +161,7 @@ func delete(app *cli.Context, cfg *cmds.Server) error {
 	if err != nil {
 		return err
 	}
-	r, err := info.Post("/db/snapshot", b)
+	r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout))
 	if err != nil {
 		return wrapServerError(err)
 	}
@@ -206,7 +216,7 @@ func list(app *cli.Context, cfg *cmds.Server) error {
 	if err != nil {
 		return err
 	}
-	r, err := info.Post("/db/snapshot", b)
+	r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout))
 	if err != nil {
 		return wrapServerError(err)
 	}
@@ -269,7 +279,7 @@ func prune(app *cli.Context, cfg *cmds.Server) error {
 	if err != nil {
 		return err
 	}
-	r, err := info.Post("/db/snapshot", b)
+	r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout))
 	if err != nil {
 		return wrapServerError(err)
 	}

pkg/etcd/etcd.go

@@ -664,12 +664,18 @@ func (e *ETCD) setName(force bool) error {
 
 // handler wraps the handler with routes for database info
 func (e *ETCD) handler(next http.Handler) http.Handler {
-	mux := mux.NewRouter().SkipClean(true)
-	mux.Use(auth.Middleware(e.config, version.Program+":server"))
-	mux.Handle("/db/info", e.infoHandler())
-	mux.Handle("/db/snapshot", e.snapshotHandler())
-	mux.NotFoundHandler = next
-	return mux
+	r := mux.NewRouter().SkipClean(true)
+	r.NotFoundHandler = next
+
+	ir := r.Path("/db/info").Subrouter()
+	ir.Use(auth.IsLocalOrHasRole(e.config, version.Program+":server"))
+	ir.Handle("", e.infoHandler())
+
+	sr := r.Path("/db/snapshot").Subrouter()
+	sr.Use(auth.HasRole(e.config, version.Program+":server"))
+	sr.Handle("", e.snapshotHandler())
+
+	return r
 }
 
 // infoHandler returns etcd cluster information. This is used by new members when joining the cluster.

pkg/etcd/snapshot_handler.go

@@ -170,14 +170,15 @@ func (e *ETCD) withRequest(sr *SnapshotRequest) *ETCD {
 	}
 	if sr.S3 != nil {
 		re.config.EtcdS3 = true
-		re.config.EtcdS3BucketName = sr.S3.Bucket
 		re.config.EtcdS3AccessKey = sr.S3.AccessKey
-		re.config.EtcdS3SecretKey = sr.S3.SecretKey
+		re.config.EtcdS3BucketName = sr.S3.Bucket
 		re.config.EtcdS3Endpoint = sr.S3.Endpoint
 		re.config.EtcdS3EndpointCA = sr.S3.EndpointCA
-		re.config.EtcdS3SkipSSLVerify = sr.S3.SkipSSLVerify
+		re.config.EtcdS3Folder = sr.S3.Folder
 		re.config.EtcdS3Insecure = sr.S3.Insecure
 		re.config.EtcdS3Region = sr.S3.Region
+		re.config.EtcdS3SecretKey = sr.S3.SecretKey
+		re.config.EtcdS3SkipSSLVerify = sr.S3.SkipSSLVerify
 		re.config.EtcdS3Timeout = sr.S3.Timeout.Duration
 	}
 	return re

pkg/server/auth/auth.go

@@ -1,6 +1,7 @@
 package auth
 
 import (
+	"net"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -22,6 +23,7 @@ func hasRole(mustRoles []string, roles []string) bool {
 	return false
 }
 
+// doAuth calls the cluster's authenticator to validate that the client has at least one of the listed roles
 func doAuth(roles []string, serverConfig *config.Control, next http.Handler, rw http.ResponseWriter, req *http.Request) {
 	switch {
 	case serverConfig == nil:
@@ -51,10 +53,27 @@ func doAuth(roles []string, serverConfig *config.Control, next http.Handler, rw
 	next.ServeHTTP(rw, req)
 }
 
-func Middleware(serverConfig *config.Control, roles ...string) mux.MiddlewareFunc {
+// HasRole returns a middleware function that validates that the request
+// is being made with at least one of the listed roles.
+func HasRole(serverConfig *config.Control, roles ...string) mux.MiddlewareFunc {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
 			doAuth(roles, serverConfig, next, rw, req)
 		})
 	}
 }
+
+// IsLocalOrHasRole returns a middleware function that validates that the request
+// is from a local client or has at least one of the listed roles.
+func IsLocalOrHasRole(serverConfig *config.Control, roles ...string) mux.MiddlewareFunc {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+			client, _, _ := net.SplitHostPort(req.RemoteAddr)
+			if client == "127.0.0.1" || client == "::1" {
+				next.ServeHTTP(rw, req)
+			} else {
+				doAuth(roles, serverConfig, next, rw, req)
+			}
+		})
+	}
+}

pkg/server/router.go

@@ -52,7 +52,7 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
 
 	prefix := "/v1-" + version.Program
 	authed := mux.NewRouter().SkipClean(true)
-	authed.Use(auth.Middleware(serverConfig, version.Program+":agent", user.NodesGroup, bootstrapapi.BootstrapDefaultGroup))
+	authed.Use(auth.HasRole(serverConfig, version.Program+":agent", user.NodesGroup, bootstrapapi.BootstrapDefaultGroup))
 	authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, nodeAuth))
 	authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, nodeAuth))
 	authed.Path(prefix + "/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert, serverConfig.Runtime.ClientKubeProxyKey))
@@ -71,12 +71,12 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
 
 	nodeAuthed := mux.NewRouter().SkipClean(true)
 	nodeAuthed.NotFoundHandler = authed
-	nodeAuthed.Use(auth.Middleware(serverConfig, user.NodesGroup))
+	nodeAuthed.Use(auth.HasRole(serverConfig, user.NodesGroup))
 	nodeAuthed.Path(prefix + "/connect").Handler(serverConfig.Runtime.Tunnel)
 
 	serverAuthed := mux.NewRouter().SkipClean(true)
 	serverAuthed.NotFoundHandler = nodeAuthed
-	serverAuthed.Use(auth.Middleware(serverConfig, version.Program+":server"))
+	serverAuthed.Use(auth.HasRole(serverConfig, version.Program+":server"))
 	serverAuthed.Path(prefix + "/encrypt/status").Handler(encryptionStatusHandler(serverConfig))
 	serverAuthed.Path(prefix + "/encrypt/config").Handler(encryptionConfigHandler(ctx, serverConfig))
 	serverAuthed.Path(prefix + "/cert/cacerts").Handler(caCertReplaceHandler(serverConfig))
@@ -86,7 +86,7 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
 	systemAuthed := mux.NewRouter().SkipClean(true)
 	systemAuthed.NotFoundHandler = serverAuthed
 	systemAuthed.MethodNotAllowedHandler = serverAuthed
-	systemAuthed.Use(auth.Middleware(serverConfig, user.SystemPrivilegedGroup))
+	systemAuthed.Use(auth.HasRole(serverConfig, user.SystemPrivilegedGroup))
 	systemAuthed.Methods(http.MethodConnect).Handler(serverConfig.Runtime.Tunnel)
 
 	staticDir := filepath.Join(serverConfig.DataDir, "static")

tests/e2e/dualstack/Vagrantfile

@@ -90,6 +90,8 @@ Vagrant.configure("2") do |config|
   config.vm.provider "libvirt" do |v|
     v.cpus = NODE_CPUS
     v.memory = NODE_MEMORY
+    # We replicate the default prefix, but add a timestamp to enable parallel runs and cleanup of old VMs
+    v.default_prefix = File.basename(Dir.getwd) + "_" + Time.now.to_i.to_s + "_"
   end
 
   if NODE_ROLES.kind_of?(String)

tests/e2e/embeddedmirror/Vagrantfile

@@ -97,6 +97,8 @@ Vagrant.configure("2") do |config|
   config.vm.provider "libvirt" do |v|
     v.cpus = NODE_CPUS
     v.memory = NODE_MEMORY
+    # We replicate the default prefix, but add a timestamp to enable parallel runs and cleanup of old VMs
+    v.default_prefix = File.basename(Dir.getwd) + "_" + Time.now.to_i.to_s + "_"
   end
   config.vm.provider "virtualbox" do |v|
     v.cpus = NODE_CPUS

tests/e2e/externalip/Vagrantfile

@@ -65,6 +65,8 @@ Vagrant.configure("2") do |config|
   config.vm.provider "libvirt" do |v|
     v.cpus = NODE_CPUS
     v.memory = NODE_MEMORY
+    # We replicate the default prefix, but add a timestamp to enable parallel runs and cleanup of old VMs
+    v.default_prefix = File.basename(Dir.getwd) + "_" + Time.now.to_i.to_s + "_"
   end
 
   if NODE_ROLES.kind_of?(String)

tests/e2e/privateregistry/Vagrantfile

@@ -103,6 +103,8 @@ Vagrant.configure("2") do |config|
   config.vm.provider "libvirt" do |v|
     v.cpus = NODE_CPUS
     v.memory = NODE_MEMORY
+    # We replicate the default prefix, but add a timestamp to enable parallel runs and cleanup of old VMs
+    v.default_prefix = File.basename(Dir.getwd) + "_" + Time.now.to_i.to_s + "_"
   end
   config.vm.provider "virtualbox" do |v|
     v.cpus = NODE_CPUS

tests/e2e/rootless/Vagrantfile

@@ -45,6 +45,8 @@ Vagrant.configure("2") do |config|
   config.vm.provider "libvirt" do |v|
     v.cpus = NODE_CPUS
     v.memory = NODE_MEMORY
+    # We replicate the default prefix, but add a timestamp to enable parallel runs and cleanup of old VMs
+    v.default_prefix = File.basename(Dir.getwd) + "_" + Time.now.to_i.to_s + "_"
   end
   config.vm.provider "virtualbox" do |v|
     v.cpus = NODE_CPUS

tests/e2e/rotateca/Vagrantfile

@@ -65,6 +65,8 @@ Vagrant.configure("2") do |config|
   config.vm.provider "libvirt" do |v|
     v.cpus = NODE_CPUS
     v.memory = NODE_MEMORY
+    # We replicate the default prefix, but add a timestamp to enable parallel runs and cleanup of old VMs
+    v.default_prefix = File.basename(Dir.getwd) + "_" + Time.now.to_i.to_s + "_"
   end
   config.vm.provider "virtualbox" do |v|
     v.cpus = NODE_CPUS

tests/e2e/s3/Vagrantfile

@@ -29,7 +29,7 @@ def provision(vm, role, role_num, node_num)
 
 
   runS3mock = <<~'SCRIPT'
-    docker run -p 9090:9090 -p 9191:9191 -d -e initialBuckets=test -e debug=true -t adobe/s3mock
+    docker run -p 9090:9090 -p 9191:9191 -d -e initialBuckets=test-bucket -e debug=true -t adobe/s3mock
   SCRIPT
 
 
@@ -45,7 +45,8 @@ def provision(vm, role, role_num, node_num)
         flannel-iface: eth1
         cluster-init: true
         etcd-s3-insecure: true
-        etcd-s3-bucket: test
+        etcd-s3-bucket: test-bucket
+        etcd-s3-folder: test-folder
         etcd-s3: true
         etcd-s3-endpoint: localhost:9090
         etcd-s3-skip-ssl-verify: true
@@ -68,6 +69,8 @@ Vagrant.configure("2") do |config|
   config.vm.provider "libvirt" do |v|
     v.cpus = NODE_CPUS
     v.memory = NODE_MEMORY
+    # We replicate the default prefix, but add a timestamp to enable parallel runs and cleanup of old VMs
+    v.default_prefix = File.basename(Dir.getwd) + "_" + Time.now.to_i.to_s + "_"
   end
   config.vm.provider "virtualbox" do |v|
     v.cpus = NODE_CPUS

tests/e2e/s3/s3_test.go

@@ -95,6 +95,7 @@ var _ = Describe("Verify Create", Ordered, func() {
 			res, err := e2e.RunCmdOnNode("k3s etcd-snapshot list", serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
 			Expect(res).To(ContainSubstring("file:///var/lib/rancher/k3s/server/db/snapshots/on-demand-server-0"))
+			Expect(res).To(ContainSubstring("s3://test-bucket/test-folder/on-demand-server-0"))
 		})
 		It("save 3 more s3 snapshots", func() {
 			for _, i := range []string{"1", "2", "3"} {
@@ -106,10 +107,10 @@ var _ = Describe("Verify Create", Ordered, func() {
 		It("lists saved s3 snapshot", func() {
 			res, err := e2e.RunCmdOnNode("k3s etcd-snapshot list", serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
-			Expect(res).To(ContainSubstring("on-demand-server-0"))
-			Expect(res).To(ContainSubstring("special-1-server-0"))
-			Expect(res).To(ContainSubstring("special-2-server-0"))
-			Expect(res).To(ContainSubstring("special-3-server-0"))
+			Expect(res).To(ContainSubstring("s3://test-bucket/test-folder/on-demand-server-0"))
+			Expect(res).To(ContainSubstring("s3://test-bucket/test-folder/special-1-server-0"))
+			Expect(res).To(ContainSubstring("s3://test-bucket/test-folder/special-2-server-0"))
+			Expect(res).To(ContainSubstring("s3://test-bucket/test-folder/special-3-server-0"))
 		})
 		It("delete first on-demand s3 snapshot", func() {
 			_, err := e2e.RunCmdOnNode("sudo k3s etcd-snapshot ls >> ./snapshotname.txt", serverNodeNames[0])