diff --git a/pkg/controller/secrets/tenant_controller.go b/pkg/controller/secrets/tenant_controller.go index ce877fa1dd..95dd1da8d0 100644 --- a/pkg/controller/secrets/tenant_controller.go +++ b/pkg/controller/secrets/tenant_controller.go @@ -227,9 +227,8 @@ func (r *TenantController) upstreamCertificates(cm certificatemanager.Certificat } if r.elasticExternal { - // If configured to use external Elasticsearch, get the Elasticsearch and Kibana public certs. + // If configured to use external Elasticsearch, get the Elasticsearch public certs. toQuery[logstorage.ExternalESPublicCertName] = common.OperatorNamespace() - toQuery[logstorage.ExternalKBPublicCertName] = common.OperatorNamespace() } // Query each certificate. diff --git a/pkg/controller/secrets/tenant_controller_test.go b/pkg/controller/secrets/tenant_controller_test.go index 77223d5cd6..996f416648 100644 --- a/pkg/controller/secrets/tenant_controller_test.go +++ b/pkg/controller/secrets/tenant_controller_test.go @@ -15,6 +15,7 @@ package secrets import ( + "bytes" "context" . "github.com/onsi/ginkgo" @@ -31,6 +32,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -41,8 +43,10 @@ import ( "github.com/tigera/operator/pkg/controller/status" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + rmeta "github.com/tigera/operator/pkg/render/common/meta" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/render/logstorage" + "github.com/tigera/operator/pkg/tls" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -112,9 +116,16 @@ var _ = Describe("Tenant controller", func() { Expect(cli.Create(ctx, cm.KeyPair().Secret(common.OperatorNamespace()))).ShouldNot(HaveOccurred()) // Create the external ES and Kibana public certificates, used for external ES. - externalESSecret := rtest.CreateCertSecret(logstorage.ExternalESPublicCertName, common.OperatorNamespace(), "external.es.com") + + cryptoCA, _ := tls.MakeCA(rmeta.TigeraOperatorCAIssuerPrefix + "@some-hash") + dnsNames := []string{"external.ingress.elastic"} + cfg, _ := cryptoCA.MakeServerCertForDuration(sets.NewString(dnsNames...), tls.DefaultCertificateDuration, tls.SetServerAuth, tls.SetClientAuth) + keyContent, crtContent := &bytes.Buffer{}, &bytes.Buffer{} + _ = cfg.WriteCertConfig(crtContent, keyContent) + + externalESSecret := rtest.CreateCertSecretWithContent(logstorage.ExternalESPublicCertName, common.OperatorNamespace(), keyContent.Bytes(), crtContent.Bytes()) Expect(cli.Create(ctx, externalESSecret)).ShouldNot(HaveOccurred()) - externalKibanaSecret := rtest.CreateCertSecret(logstorage.ExternalKBPublicCertName, common.OperatorNamespace(), "external.kb.com") + externalKibanaSecret := rtest.CreateCertSecretWithContent(logstorage.ExternalKBPublicCertName, common.OperatorNamespace(), keyContent.Bytes(), crtContent.Bytes()) Expect(cli.Create(ctx, externalKibanaSecret)).ShouldNot(HaveOccurred()) // Create the tenant Namespace. @@ -174,9 +185,8 @@ var _ = Describe("Tenant controller", func() { types.NamespacedName{Name: certificatemanagement.CASecretName, Namespace: common.OperatorNamespace()}, types.NamespacedName{Name: certificatemanagement.TenantCASecretName, Namespace: tenantNS}, - // Should include public certs for external ES and Kibana. + // Should include public certs for external ES. types.NamespacedName{Name: logstorage.ExternalESPublicCertName, Namespace: common.OperatorNamespace()}, - types.NamespacedName{Name: logstorage.ExternalKBPublicCertName, Namespace: common.OperatorNamespace()}, ) // A trusted bundle ConfigMap with system roots should also have been created. diff --git a/pkg/render/common/test/testing.go b/pkg/render/common/test/testing.go index 12de6c7fe3..ea48659d95 100644 --- a/pkg/render/common/test/testing.go +++ b/pkg/render/common/test/testing.go @@ -283,6 +283,20 @@ func CreateCertSecret(name, namespace string, dnsNames ...string) *corev1.Secret } } +func CreateCertSecretWithContent(name, namespace string, keyContent []byte, crtContent []byte) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Data: map[string][]byte{ + corev1.TLSPrivateKeyKey: keyContent, + corev1.TLSCertKey: crtContent, + }, + } +} + func ExpectBundleContents(bundle *corev1.ConfigMap, secrets ...types.NamespacedName) { ExpectWithOffset(1, bundle.Data).To(HaveKey("tigera-ca-bundle.crt"), fmt.Sprintf("Bundle: %+v", bundle)) diff --git a/pkg/render/manager/manager_route_config.go b/pkg/render/manager/manager_route_config.go index 1d1f9d931d..e9329899d0 100644 --- a/pkg/render/manager/manager_route_config.go +++ b/pkg/render/manager/manager_route_config.go @@ -290,6 +290,9 @@ func (builder *voltronRouteConfigBuilder) Build() (*VoltronRouteConfig, error) { return nil, err } + sort.Sort(ByVolumeMountName(builder.volumeMounts)) + sort.Sort(ByVolumeName(builder.volumes)) + return &VoltronRouteConfig{ routesData: routesData, volumeMounts: builder.volumeMounts, @@ -459,6 +462,18 @@ type VoltronRouteConfig struct { annotations map[string]string } +type ByVolumeMountName []corev1.VolumeMount + +func (m ByVolumeMountName) Len() int { return len(m) } +func (m ByVolumeMountName) Less(i, j int) bool { return m[i].Name < m[j].Name } +func (m ByVolumeMountName) Swap(i, j int) { m[i], m[j] = m[j], m[i] } + +type ByVolumeName []corev1.Volume + +func (m ByVolumeName) Len() int { return len(m) } +func (m ByVolumeName) Less(i, j int) bool { return m[i].Name < m[j].Name } +func (m ByVolumeName) Swap(i, j int) { m[i], m[j] = m[j], m[i] } + // Volumes returns the volumes that Voltron needs to be configured with (references to ConfigMaps and Secrets in the // TLSTerminatedRoute CRs). func (cfg *VoltronRouteConfig) Volumes() []corev1.Volume { diff --git a/pkg/render/manager/manager_route_config_test.go b/pkg/render/manager/manager_route_config_test.go index 15f1b2f79b..3d74157877 100644 --- a/pkg/render/manager/manager_route_config_test.go +++ b/pkg/render/manager/manager_route_config_test.go @@ -345,9 +345,9 @@ var _ = Describe("VoltronRouteConfigBuilder", func() { "hash.operator.tigera.io/routeconf-s-mtls-key-key.pem": "6b519c7eea53167b5fe03c86b7650ada4e7a4784", routeCMKey: "907bc0d66a81235ae423c36bda0ed50fa73f7f51", })) - Expect(config.VolumeMounts()).Should(Equal([]corev1.VolumeMount{caBundleVolumeMount, mtlsCertVolumeMount, mtlsKeyVolumeMount, routesConfigMapVolumeMount})) + Expect(config.VolumeMounts()).Should(Equal([]corev1.VolumeMount{caBundleVolumeMount, routesConfigMapVolumeMount, mtlsCertVolumeMount, mtlsKeyVolumeMount})) - Expect(config.Volumes()).Should(Equal([]corev1.Volume{caBundleVolume, mtlsCertVolume, mtlsKeyVolume, routesConfigMapVolume})) + Expect(config.Volumes()).Should(Equal([]corev1.Volume{caBundleVolume, routesConfigMapVolume, mtlsCertVolume, mtlsKeyVolume})) cm := config.RoutesConfigMap("tigera-manager") cm.Data[fileName] = compactJSONString(cm.Data[fileName]) @@ -359,6 +359,111 @@ var _ = Describe("VoltronRouteConfigBuilder", func() { Entry("Upstream tunnel target", operatorv1.TargetTypeUpstreamTunnel, "upTunTLSTermRoutes.json", "hash.operator.tigera.io/routeconf-cm-voltron-routes-uptuntlster"), ) }) + + When("adding multiple routes out of order", func() { + It("volume and volume mounts should be sorted", func() { + routes := []operatorv1.TLSTerminatedRoute{ + { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "route-2"}, + Spec: operatorv1.TLSTerminatedRouteSpec{ + Target: operatorv1.TargetTypeUI, + CABundle: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "ca-bundle", + }, + Key: "ca-bundle.crt", + }, + PathMatch: &operatorv1.PathMatch{ + Path: "/bar/", + PathRegexp: ptr.ToPtr("^/bar/?"), + PathReplace: ptr.ToPtr("/"), + }, + Destination: "bar", + }, + }, + { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "route-1"}, + Spec: operatorv1.TLSTerminatedRouteSpec{ + Target: operatorv1.TargetTypeUI, + CABundle: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "public-cert", + }, + Key: "tls.crt", + }, + PathMatch: &operatorv1.PathMatch{ + Path: "/foo/", + PathRegexp: ptr.ToPtr("^/foo/?"), + PathReplace: ptr.ToPtr("/"), + }, + Destination: "foo", + }, + }, + { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "route-3"}, + Spec: operatorv1.TLSTerminatedRouteSpec{ + Target: operatorv1.TargetTypeUI, + CABundle: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "ca-bundle", + }, + Key: "ca-bundle.crt", + }, + PathMatch: &operatorv1.PathMatch{ + Path: "/goo/", + PathRegexp: ptr.ToPtr("^/goo/?"), + PathReplace: ptr.ToPtr("/"), + }, + Destination: "goo", + }, + }, + } + for _, route := range routes { + builder.AddTLSTerminatedRoute(route) + } + + builder.AddConfigMap(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "public-cert", + }, + Data: map[string]string{ + "tls.crt": "bundle", + }, + }) + builder.AddConfigMap(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ca-bundle", + }, + Data: map[string]string{ + "ca-bundle.crt": "bundle", + }, + }) + config, err := builder.Build() + Expect(err).ShouldNot(HaveOccurred()) + + Expect(config.VolumeMounts()).Should(Equal([]corev1.VolumeMount{ + { + Name: "cm-ca-bundle", + MountPath: "/config_maps/ca-bundle", + ReadOnly: true, + }, + { + Name: "cm-public-cert", + MountPath: "/config_maps/public-cert", + ReadOnly: true, + }, + { + Name: "cm-voltron-routes", + MountPath: "/config_maps/voltron-routes", + ReadOnly: true, + }, + })) + + }) + }) }) })