From 865ffd1cbd13d5b549dc8084ffed3f6e7ee40d7b Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Tue, 12 Nov 2024 10:32:14 +0100 Subject: [PATCH] Test runtimextension integration Signed-off-by: Danil-Grigorev --- Makefile | 1 + test/e2e/cluster_upgrade_runtimesdk.go | 33 +++++++++++++++---- test/e2e/cluster_upgrade_runtimesdk_test.go | 33 +++++++++++++++++++ test/e2e/config/docker.yaml | 1 + .../cluster.yaml | 8 +++++ .../kustomization.yaml | 5 +++ 6 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 test/e2e/data/infrastructure-docker/main/cluster-template-cross-ns-upgrades-runtimesdk/cluster.yaml create mode 100644 test/e2e/data/infrastructure-docker/main/cluster-template-cross-ns-upgrades-runtimesdk/kustomization.yaml diff --git a/Makefile b/Makefile index e27b07ac0740..cfb1734f074b 100644 --- a/Makefile +++ b/Makefile @@ -618,6 +618,7 @@ generate-e2e-templates-main: $(KUSTOMIZE) $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-ignition --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-ignition.yaml $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/clusterclass-quick-start-kcp-only --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/clusterclass-quick-start-kcp-only.yaml $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-cross-ns-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-cross-ns-topology.yaml + $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-cross-ns-upgrades-runtimesdk --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-cross-ns-upgrades-runtimesdk.yaml $(KUSTOMIZE) build $(INMEMORY_TEMPLATES)/main/cluster-template --load-restrictor LoadRestrictionsNone > $(INMEMORY_TEMPLATES)/main/cluster-template.yaml .PHONY: generate-metrics-config diff --git a/test/e2e/cluster_upgrade_runtimesdk.go b/test/e2e/cluster_upgrade_runtimesdk.go index cbbb04da1f8a..5cbcb933347c 100644 --- a/test/e2e/cluster_upgrade_runtimesdk.go +++ b/test/e2e/cluster_upgrade_runtimesdk.go @@ -95,6 +95,9 @@ type ClusterUpgradeWithRuntimeSDKSpecInput struct { ExtensionServiceNamespace string // ExtensionServiceName is the name of the service to configure in the test-namespace scoped ExtensionConfig. ExtensionServiceName string + + // ClassNamespace is an optional class namespace reference, configuring cross-namespace cluster class reference + ClassNamespace bool } // ClusterUpgradeWithRuntimeSDKSpec implements a spec that upgrades a cluster and runs the Kubernetes conformance suite. @@ -109,9 +112,9 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl ) var ( - input ClusterUpgradeWithRuntimeSDKSpecInput - namespace *corev1.Namespace - cancelWatches context.CancelFunc + input ClusterUpgradeWithRuntimeSDKSpecInput + namespace, infraNamespace *corev1.Namespace + cancelWatches, cancelInfraWatches context.CancelFunc controlPlaneMachineCount int64 workerMachineCount int64 @@ -148,6 +151,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl // Set up a Namespace where to host objects for this spec and create a watcher for the Namespace events. namespace, cancelWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated) + infraNamespace, cancelInfraWatches = framework.SetupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, input.PostNamespaceCreated) clusterName = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) }) @@ -162,7 +166,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl By("Deploy Test Extension ExtensionConfig") Expect(input.BootstrapClusterProxy.GetClient().Create(ctx, - extensionConfig(specName, namespace.Name, input.ExtensionServiceNamespace, input.ExtensionServiceName))). + extensionConfig(specName, input.ExtensionServiceNamespace, input.ExtensionServiceName, namespace.Name, infraNamespace.Name))). To(Succeed(), "Failed to create the extension config") By("Creating a workload cluster; creation waits for BeforeClusterCreateHook to gate the operation") @@ -177,6 +181,11 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl infrastructureProvider = *input.InfrastructureProvider } + variables := map[string]string{} + if input.ClassNamespace { + variables["CLUSTER_CLASS_NAMESPACE"] = infraNamespace.Name + } + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ ClusterProxy: input.BootstrapClusterProxy, ConfigCluster: clusterctl.ConfigClusterInput{ @@ -190,6 +199,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom), ControlPlaneMachineCount: ptr.To[int64](controlPlaneMachineCount), WorkerMachineCount: ptr.To[int64](workerMachineCount), + ClusterctlVariables: variables, }, PreWaitForCluster: func() { beforeClusterCreateTestHandler(ctx, @@ -304,7 +314,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl if !input.SkipCleanup { // Delete the extensionConfig first to ensure the BeforeDeleteCluster hook doesn't block deletion. Eventually(func() error { - return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(specName, namespace.Name, input.ExtensionServiceNamespace, input.ExtensionServiceName)) + return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(specName, input.ExtensionServiceNamespace, input.ExtensionServiceName)) }, 10*time.Second, 1*time.Second).Should(Succeed(), "delete extensionConfig failed") Byf("Deleting cluster %s", klog.KObj(clusterResources.Cluster)) @@ -322,8 +332,17 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl Deleter: input.BootstrapClusterProxy.GetClient(), Name: namespace.Name, }) + + if input.ClassNamespace { + Byf("Deleting namespace used for optionally hosting the %q infrastructure spec", specName) + framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{ + Deleter: input.BootstrapClusterProxy.GetClient(), + Name: infraNamespace.Name, + }) + } } cancelWatches() + cancelInfraWatches() }) } @@ -429,7 +448,7 @@ func machineSetPreflightChecksTestHandler(ctx context.Context, c client.Client, // We make sure this cluster-wide object does not conflict with others by using a random generated // name and a NamespaceSelector selecting on the namespace of the current test. // Thus, this object is "namespaced" to the current test even though it's a cluster-wide object. -func extensionConfig(name, namespace, extensionServiceNamespace, extensionServiceName string) *runtimev1.ExtensionConfig { +func extensionConfig(name, extensionServiceNamespace, extensionServiceName string, namespaces ...string) *runtimev1.ExtensionConfig { return &runtimev1.ExtensionConfig{ ObjectMeta: metav1.ObjectMeta{ // Note: We have to use a constant name here as we have to be able to reference it in the ClusterClass @@ -454,7 +473,7 @@ func extensionConfig(name, namespace, extensionServiceNamespace, extensionServic { Key: "kubernetes.io/metadata.name", Operator: metav1.LabelSelectorOpIn, - Values: []string{namespace}, + Values: namespaces, }, }, }, diff --git a/test/e2e/cluster_upgrade_runtimesdk_test.go b/test/e2e/cluster_upgrade_runtimesdk_test.go index 3628dce38133..e87247dac609 100644 --- a/test/e2e/cluster_upgrade_runtimesdk_test.go +++ b/test/e2e/cluster_upgrade_runtimesdk_test.go @@ -60,3 +60,36 @@ var _ = Describe("When upgrading a workload cluster using ClusterClass with Runt } }) }) + +var _ = Describe("When upgrading a workload cluster using ClusterClass in a different NS with RuntimeSDK [PR-Blocking] [ClusterClass]", func() { + ClusterUpgradeWithRuntimeSDKSpec(ctx, func() ClusterUpgradeWithRuntimeSDKSpecInput { + version, err := semver.ParseTolerant(e2eConfig.GetVariable(KubernetesVersionUpgradeFrom)) + Expect(err).ToNot(HaveOccurred(), "Invalid argument, KUBERNETES_VERSION_UPGRADE_FROM is not a valid version") + if version.LT(semver.MustParse("1.24.0")) { + Fail("This test only supports upgrades from Kubernetes >= v1.24.0") + } + + return ClusterUpgradeWithRuntimeSDKSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + InfrastructureProvider: ptr.To("docker"), + PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) { + // This check ensures that the resourceVersions are stable, i.e. it verifies there are no + // continuous reconciles when everything should be stable. + framework.ValidateResourceVersionStable(ctx, proxy, namespace, clusterctlcluster.FilterClusterObjectsWithNameFilter(clusterName)) + }, + // "upgrades" is the same as the "topology" flavor but with an additional MachinePool. + Flavor: ptr.To("cross-ns-upgrades-runtimesdk"), + ClassNamespace: true, + // The runtime extension gets deployed to the test-extension-system namespace and is exposed + // by the test-extension-webhook-service. + // The below values are used when creating the cluster-wide ExtensionConfig to refer + // the actual service. + ExtensionServiceNamespace: "test-extension-system", + ExtensionServiceName: "test-extension-webhook-service", + } + }) +}) diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml index 737b3b7d7c67..591f97c2f21f 100644 --- a/test/e2e/config/docker.yaml +++ b/test/e2e/config/docker.yaml @@ -348,6 +348,7 @@ providers: - sourcePath: "../data/infrastructure-docker/main/cluster-template-topology.yaml" - sourcePath: "../data/infrastructure-docker/main/cluster-template-ignition.yaml" - sourcePath: "../data/infrastructure-docker/main/cluster-template-cross-ns-topology.yaml" + - sourcePath: "../data/infrastructure-docker/main/cluster-template-cross-ns-upgrades-runtimesdk.yaml" - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start.yaml" - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-kcp-only.yaml" - sourcePath: "../data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk.yaml" diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-cross-ns-upgrades-runtimesdk/cluster.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-cross-ns-upgrades-runtimesdk/cluster.yaml new file mode 100644 index 000000000000..066f8b174900 --- /dev/null +++ b/test/e2e/data/infrastructure-docker/main/cluster-template-cross-ns-upgrades-runtimesdk/cluster.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: '${CLUSTER_NAME}' +spec: + topology: + classNamespace: '${CLUSTER_CLASS_NAMESPACE}' diff --git a/test/e2e/data/infrastructure-docker/main/cluster-template-cross-ns-upgrades-runtimesdk/kustomization.yaml b/test/e2e/data/infrastructure-docker/main/cluster-template-cross-ns-upgrades-runtimesdk/kustomization.yaml new file mode 100644 index 000000000000..df18913aca6e --- /dev/null +++ b/test/e2e/data/infrastructure-docker/main/cluster-template-cross-ns-upgrades-runtimesdk/kustomization.yaml @@ -0,0 +1,5 @@ +resources: + - ../cluster-template-upgrades-runtimesdk + +patches: +- path: cluster.yaml \ No newline at end of file