feat: allow Secrets as BackendTLSPolicy CACertificateRefs (#6853)

* feat: BackendTLSPolicy secret CA certificates Signed-off-by: Mattia Lavacca <[email protected]> * test: improve unit Signed-off-by: Mattia Lavacca <[email protected]> * chore: update CHANGELOG Signed-off-by: Mattia Lavacca <[email protected]> * chore: make manifests Signed-off-by: Mattia Lavacca <[email protected]> * test: integration test improved Signed-off-by: Mattia Lavacca <[email protected]> * fix: no ingress-class annotation on CA secretes Signed-off-by: Mattia Lavacca <[email protected]> * address reviews comments Signed-off-by: Mattia Lavacca <[email protected]> --------- Signed-off-by: Mattia Lavacca <[email protected]>
Kong · Dec 17, 2024 · d8e83b4 · d8e83b4
1 parent f8a7606
commit d8e83b4
Show file tree

Hide file tree

Showing 16 changed files with 271 additions and 59 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -254,10 +254,12 @@ Adding a new version? You'll need three changes:
 - Added `BackendTLSPolicy` support. The user can now reference any Kubernetes `Service`
   in the `BackendTLSPolicy` spec, and in case the service is used as a backend by
   `HTTPRoute`s that reference a Kong Gateway as parent, such Backend TLS configuration
-  is applied to the service section of the Kong configuration.
+  is applied to the service section of the Kong configuration. The `BackendTLSPolicies`
+  CA Certificates can be set in `Secret`s or `ConfigMap`s.
   [#6712](https://github.com/Kong/kubernetes-ingress-controller/pull/6712)
   [#6753](https://github.com/Kong/kubernetes-ingress-controller/pull/6753)
   [#6837](https://github.com/Kong/kubernetes-ingress-controller/pull/6837)
+  [#6853](https://github.com/Kong/kubernetes-ingress-controller/pull/6853)
 - Added the flag `--configmap-label-selector` to set the label selector for `ConfigMap`s
   to ingest. By setting this flag, the `ConfigMap`s that are ingested will be limited
   to those having this label set to "true". This limits the amount of resources that are kept in memory.

diff --git a/config/rbac/gateway/role.yaml b/config/rbac/gateway/role.yaml
@@ -9,6 +9,7 @@ rules:
   resources:
   - configmaps
   - namespaces
+  - secrets
   - services
   verbs:
   - get

diff --git a/internal/annotations/annotations.go b/internal/annotations/annotations.go
@@ -455,13 +455,29 @@ func SetTLSVerify(anns map[string]string, value bool) {
 	anns[AnnotationPrefix+TLSVerifyKey] = strconv.FormatBool(value)
 }
 
-// SetCACertificates merge the ca-certificates secret names into the already existing CA certificates set via annotation.
-func SetCACertificates(anns map[string]string, certificates []string) {
+// SetConfigMapCACertificates merge the ca-certificates configmap names into the already existing CA certificates set via annotation.
+func SetConfigMapCACertificates(anns map[string]string, configMapCertificates []string) {
+	if len(configMapCertificates) == 0 {
+		return
+	}
 	existingCACerts := anns[AnnotationPrefix+CACertificatesConfigMapsKey]
 	if existingCACerts == "" {
-		anns[AnnotationPrefix+CACertificatesConfigMapsKey] = strings.Join(certificates, ",")
+		anns[AnnotationPrefix+CACertificatesConfigMapsKey] = strings.Join(configMapCertificates, ",")
+	} else {
+		anns[AnnotationPrefix+CACertificatesConfigMapsKey] = existingCACerts + "," + strings.Join(configMapCertificates, ",")
+	}
+}
+
+// SetSecretCACertificates merge the ca-certificates secret names into the already existing CA certificates set via annotation.
+func SetSecretCACertificates(anns map[string]string, secretCertificates []string) {
+	if len(secretCertificates) == 0 {
+		return
+	}
+	existingCACerts := anns[AnnotationPrefix+CACertificatesSecretsKey]
+	if existingCACerts == "" {
+		anns[AnnotationPrefix+CACertificatesSecretsKey] = strings.Join(secretCertificates, ",")
 	} else {
-		anns[AnnotationPrefix+CACertificatesConfigMapsKey] = existingCACerts + "," + strings.Join(certificates, ",")
+		anns[AnnotationPrefix+CACertificatesSecretsKey] = existingCACerts + "," + strings.Join(secretCertificates, ",")
 	}
 }
 

diff --git a/internal/controllers/gateway/backendtlspolicy_controller.go b/internal/controllers/gateway/backendtlspolicy_controller.go
@@ -55,6 +55,7 @@ func (r *BackendTLSPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		}).
 		For(&gatewayapi.BackendTLSPolicy{}).
 		Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(r.listBackendTLSPoliciesForConfigMaps)).
+		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(r.listBackendTLSPoliciesForSecrets)).
 		Watches(&corev1.Service{}, handler.EnqueueRequestsFromMapFunc(r.listBackendTLSPoliciesForServices)).
 		Watches(&gatewayapi.HTTPRoute{}, handler.EnqueueRequestsFromMapFunc(r.listBackendTLSPoliciesForHTTPRoutes)).
 		Watches(&gatewayapi.Gateway{}, handler.EnqueueRequestsFromMapFunc(r.listBackendTLSPoliciesForGateways)).
@@ -66,9 +67,12 @@ func (r *BackendTLSPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error {
 // -----------------------------------------------------------------------------
 
 const (
-	// backendTLSPolicyValidationCARefIndexKey is the index key for BackendTLSPolicy objects by their validation CA configmap reference.
+	// backendTLSPolicyValidationCARefConfigMapIndexKey is the index key for BackendTLSPolicy objects by their validation CA configmap reference.
 	// The value is the name of the configmap.
-	backendTLSPolicyValidationCARefIndexKey = "backendtlspolicy-validation-cacertificateref"
+	backendTLSPolicyValidationCARefConfigMapIndexKey = "backendtlspolicy-validation-cacertificateref-configmap"
+	// backendTLSPolicyValidationCARefSecretIndexKey is the index key for BackendTLSPolicy objects by their validation CA secret reference.
+	// The value is the name of the secret.
+	backendTLSPolicyValidationCARefSecretIndexKey = "backendtlspolicy-validation-cacertificateref-secret"
 	// backendTLSPolicyTargetRefIndexKey is the index key for BackendTLSPolicy objects by their target service reference.
 	// The value is the name of the service.
 	backendTLSPolicyTargetRefIndexKey = "backendtlspolicy-targetref"
@@ -96,23 +100,40 @@ func indexBackendTLSPolicyOnTargetRef(obj client.Object) []string {
 	return services
 }
 
-// indexBackendTLSPolicyOnValidationCACertificateRef indexes BackendTLSPolicy objects
+// indexBackendTLSPolicyOnValidationCACertificateConfigMapRef indexes BackendTLSPolicy objects
 // by their validation CA Certificate configmap reference.
-func indexBackendTLSPolicyOnValidationCACertificateRef(obj client.Object) []string {
+func indexBackendTLSPolicyOnValidationCACertificateConfigMapRef(obj client.Object) []string {
 	policy, ok := obj.(*gatewayapi.BackendTLSPolicy)
 	if !ok {
 		return []string{}
 	}
 
 	configmaps := []string{}
 	for _, cacertref := range policy.Spec.Validation.CACertificateRefs {
-		if (cacertref.Group == "" || cacertref.Group == "core") && cacertref.Kind == "ConfigMap" {
+		if (cacertref.Group == "" || cacertref.Group == "core") && cacertref.Kind == ctrlref.KindConfigMap {
 			configmaps = append(configmaps, string(cacertref.Name))
 		}
 	}
 	return configmaps
 }
 
+// indexBackendTLSPolicyOnValidationCACertificateSecretRef indexes BackendTLSPolicy objects
+// by their validation CA Certificate secret reference.
+func indexBackendTLSPolicyOnValidationCACertificateSecretRef(obj client.Object) []string {
+	policy, ok := obj.(*gatewayapi.BackendTLSPolicy)
+	if !ok {
+		return []string{}
+	}
+
+	secrets := []string{}
+	for _, cacertref := range policy.Spec.Validation.CACertificateRefs {
+		if (cacertref.Group == "" || cacertref.Group == "core") && cacertref.Kind == ctrlref.KindSecret {
+			secrets = append(secrets, string(cacertref.Name))
+		}
+	}
+	return secrets
+}
+
 // indexHTTPRouteOnParentRef indexes HTTPRoute objects by their parent Gateway references.
 func indexHTTPRouteOnParentRef(obj client.Object) []string {
 	httpRoute, ok := obj.(*gatewayapi.HTTPRoute)
@@ -172,12 +193,21 @@ func setupBackendTLSPolicyIndices(mgr ctrl.Manager) error {
 	if err := mgr.GetCache().IndexField(
 		context.Background(),
 		&gatewayapi.BackendTLSPolicy{},
-		backendTLSPolicyValidationCARefIndexKey,
-		indexBackendTLSPolicyOnValidationCACertificateRef,
+		backendTLSPolicyValidationCARefConfigMapIndexKey,
+		indexBackendTLSPolicyOnValidationCACertificateConfigMapRef,
 	); err != nil {
 		return fmt.Errorf("failed to index backendTLSPolicies on validation CA configmap reference: %w", err)
 	}
 
+	if err := mgr.GetCache().IndexField(
+		context.Background(),
+		&gatewayapi.BackendTLSPolicy{},
+		backendTLSPolicyValidationCARefSecretIndexKey,
+		indexBackendTLSPolicyOnValidationCACertificateSecretRef,
+	); err != nil {
+		return fmt.Errorf("failed to index backendTLSPolicies on validation CA secret reference: %w", err)
+	}
+
 	if err := mgr.GetCache().IndexField(
 		context.Background(),
 		&gatewayapi.HTTPRoute{},
@@ -210,10 +240,10 @@ func (r *BackendTLSPolicyReconciler) listBackendTLSPoliciesForConfigMaps(ctx con
 		r.Log.Error(fmt.Errorf("invalid type"), "Found invalid type in event handlers", "expected", "ConfigMap", "found", reflect.TypeOf(obj))
 		return nil
 	}
-	policies := &gatewayapi.BackendTLSPolicyList{}
-	if err := r.List(ctx, policies,
+	policies := gatewayapi.BackendTLSPolicyList{}
+	if err := r.List(ctx, &policies,
 		client.InNamespace(cm.Namespace),
-		client.MatchingFields{backendTLSPolicyValidationCARefIndexKey: cm.Name},
+		client.MatchingFields{backendTLSPolicyValidationCARefConfigMapIndexKey: cm.Name},
 	); err != nil {
 		r.Log.Error(err, "Failed to list BackendTLSPolicies for ConfigMap", "configmap", cm)
 		return nil
@@ -227,6 +257,30 @@ func (r *BackendTLSPolicyReconciler) listBackendTLSPoliciesForConfigMaps(ctx con
 	return requests
 }
 
+// listBackendTLSPoliciesForConfigMaps returns the list of BackendTLSPolicies that targets the given ConfigMap.
+func (r *BackendTLSPolicyReconciler) listBackendTLSPoliciesForSecrets(ctx context.Context, obj client.Object) []reconcile.Request {
+	secret, ok := obj.(*corev1.Secret)
+	if !ok {
+		r.Log.Error(fmt.Errorf("invalid type"), "Found invalid type in event handlers", "expected", "Secret", "found", reflect.TypeOf(obj))
+		return nil
+	}
+	policies := gatewayapi.BackendTLSPolicyList{}
+	if err := r.List(ctx, &policies,
+		client.InNamespace(secret.Namespace),
+		client.MatchingFields{backendTLSPolicyValidationCARefSecretIndexKey: secret.Name},
+	); err != nil {
+		r.Log.Error(err, "Failed to list BackendTLSPolicies for Secret", "Secret", secret)
+		return nil
+	}
+	requests := make([]reconcile.Request, 0, len(policies.Items))
+	for _, policy := range policies.Items {
+		requests = append(requests, reconcile.Request{
+			NamespacedName: client.ObjectKeyFromObject(&policy),
+		})
+	}
+	return requests
+}
+
 // listBackendTLSPoliciesForServices returns the list of BackendTLSPolicies that targets the given Service.
 func (r *BackendTLSPolicyReconciler) listBackendTLSPoliciesForServices(ctx context.Context, obj client.Object) []reconcile.Request {
 	service, ok := obj.(*corev1.Service)
@@ -320,7 +374,7 @@ func (r *BackendTLSPolicyReconciler) listBackendTLSPoliciesForGateways(ctx conte
 // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes;gateways;gatewayclasses,verbs=get;list;watch
 // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=backendtlspolicies,verbs=get;list;watch
 // +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=backendtlspolicies/status,verbs=patch;update
-// +kubebuilder:rbac:groups="",resources=services;configmaps,verbs=get;list;watch
+// +kubebuilder:rbac:groups="",resources=services;configmaps;secrets,verbs=get;list;watch
 
 // Reconcile processes the watched objects.
 func (r *BackendTLSPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@@ -356,7 +410,7 @@ func (r *BackendTLSPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Req
 			}
 
 			// Update references to ConfigMaps in the dataplane cache.
-			referredConfigMapNames := listConfigMapNamesReferredByBackendTLSPolicy(backendTLSPolicy)
+			referredConfigMapNames := listCaCertNamespacedNamesReferredByBackendTLSPolicy(backendTLSPolicy, ctrlref.KindConfigMap)
 			if err := ctrlref.UpdateReferencesToSecretOrConfigMap(
 				ctx,
 				r.Client,
@@ -369,6 +423,22 @@ func (r *BackendTLSPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Req
 					return ctrl.Result{Requeue: true}, nil
 				}
 			}
+
+			// Update references to Secrets in the dataplane cache.
+			referredSecretNames := listCaCertNamespacedNamesReferredByBackendTLSPolicy(backendTLSPolicy, ctrlref.KindSecret)
+			if err := ctrlref.UpdateReferencesToSecretOrConfigMap(
+				ctx,
+				r.Client,
+				r.ReferenceIndexers,
+				r.DataplaneClient,
+				backendTLSPolicy,
+				referredSecretNames,
+				&corev1.Secret{}); err != nil {
+				if apierrors.IsNotFound(err) {
+					return ctrl.Result{Requeue: true}, nil
+				}
+			}
+
 		} else {
 			// In case the policy is not accepted, ensure it gets deleted from the dataplane cache
 			if err := r.DataplaneClient.DeleteObject(backendTLSPolicy); err != nil {

diff --git a/internal/controllers/gateway/backendtlspolicy_utils.go b/internal/controllers/gateway/backendtlspolicy_utils.go
@@ -3,6 +3,7 @@ package gateway
 import (
 	"context"
 	"fmt"
+	"reflect"
 	"sort"
 	"strings"
 
@@ -13,6 +14,7 @@ import (
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	ctrlref "github.com/kong/kubernetes-ingress-controller/v3/internal/controllers/reference"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi"
 )
 
@@ -238,21 +240,29 @@ func (r *BackendTLSPolicyReconciler) validateBackendTLSPolicy(ctx context.Contex
 
 	var invalidMessages []string
 	for _, caCert := range policy.Spec.Validation.CACertificateRefs {
-		if (caCert.Group != "core" && caCert.Group != "") || caCert.Kind != "ConfigMap" {
-			invalidMessages = append(invalidMessages, "CACertificateRefs must reference ConfigMaps in the core group")
+		if (caCert.Group != "core" && caCert.Group != "") || (caCert.Kind != ctrlref.KindConfigMap && caCert.Kind != ctrlref.KindSecret) {
+			invalidMessages = append(invalidMessages, "CACertificateRefs must reference ConfigMaps or Secrets in the core group")
 			break
 		}
 
 		var (
-			cm          corev1.ConfigMap
-			configMapNN = k8stypes.NamespacedName{
+			caCertObj   client.Object
+			caCertObjNN = k8stypes.NamespacedName{
 				Namespace: policy.Namespace,
 				Name:      string(caCert.Name),
 			}
 		)
-		if err := r.Get(ctx, configMapNN, &cm); err != nil {
+		// No need for default in this switch as if the Kind is different from Secret or ConfigMap, we never get here.
+		switch caCert.Kind {
+		case ctrlref.KindSecret:
+			caCertObj = &corev1.Secret{}
+		case ctrlref.KindConfigMap:
+			caCertObj = &corev1.ConfigMap{}
+		}
+
+		if err := r.Get(ctx, caCertObjNN, caCertObj); err != nil {
 			invalidMessages = append(invalidMessages,
-				fmt.Sprintf("failed getting ConfigMap %s set as CACertificateRef: %s", configMapNN, err),
+				fmt.Sprintf("failed getting %s %s set as CACertificateRef: %s", reflect.TypeOf(caCertObj).String(), caCertObjNN, err),
 			)
 			break
 		}
@@ -272,11 +282,13 @@ func (r *BackendTLSPolicyReconciler) validateBackendTLSPolicy(ctx context.Contex
 	return acceptedCondition, nil
 }
 
-// list namespaced names of configmaps referred by the gateway.
-func listConfigMapNamesReferredByBackendTLSPolicy(policy *gatewayapi.BackendTLSPolicy) map[k8stypes.NamespacedName]struct{} {
-	// no need to check group and kind, as if they were different from core/Configmap, the policy would have been marked as invalid.
+// list namespaced names of configmaps or secrets referred by the gateway.
+func listCaCertNamespacedNamesReferredByBackendTLSPolicy(policy *gatewayapi.BackendTLSPolicy, caCertKind gatewayapi.Kind) map[k8stypes.NamespacedName]struct{} {
 	nsNames := make(map[k8stypes.NamespacedName]struct{}, len(policy.Spec.Validation.CACertificateRefs))
 	for _, certRef := range policy.Spec.Validation.CACertificateRefs {
+		if certRef.Kind != caCertKind {
+			continue
+		}
 		nsName := k8stypes.NamespacedName{
 			Namespace: policy.Namespace,
 			Name:      string(certRef.Name),