From 557f7c37e108b1caa221979a3a7312cb679ae714 Mon Sep 17 00:00:00 2001 From: hc-github-team-consul-core Date: Wed, 21 Aug 2024 12:03:56 -0400 Subject: [PATCH] Backport of [NET-10567] Fix namespace normalization on external registration/ACL Setup for Terminating Gateways into release/1.5.x (#4259) [NET-10567] Fix namespace normalization on external registration/ACL Setup for Terminating Gateways (#4224) * fix bug in external service registration ACL creation where namespace is left emtpy in acl policy if not specified in the CRD which results in an invalid acl policy * Remove check for timestamp * update tests! * update to use helper function * all non default working * test cases all working * move wait for it to separate PR * use replace for consul-k8s control-plane * update single namespace test * updated namespaces and destinations test * remove usage of creating terminating gateway config entry creation and external service config entry registration from tests * fix typo * update comment * comment out broken test for the time being * remove unused import and add period to comment * add changelog * fix bug in cache creation for registrations, still debugging issue with termianting gateways and acl roles * fix issue with terminating gateway acl role by moving role modification from registrations controller to terminating gateway controller * appease the linter * add acl status condition to terminating gateways * linter * update config entry terminating gateway tests * Use more robust method of checking if acls are enabled * update config entries controller unit tests to run with acls and without * fix config entries namespaces test setup * fix unused import * fix config entries main test * remove block for deregistering service * fix comment * fix acceptance test registration * handle removing policies when no other gateways reference them * fix terminating gateway configuration for peering connect test * remove unnecessary nodeMeta on fixture, remove unused yaml files from fixtures * fix wildcard service names * use more specific matchers to avoid potential substring collisions * Update .changelog/4224.txt * cleaning up from PR review: moving template execution to where it's needed and updating variable names to be more consistent * add comment * fix typo --------- Co-authored-by: John Maguire Co-authored-by: Nathan Coleman --- .changelog/4224.txt | 3 + acceptance/framework/helpers/helpers.go | 132 +++-- acceptance/framework/k8s/deploy.go | 4 +- acceptance/go.mod | 16 +- acceptance/go.sum | 15 +- .../config_entries_namespaces_test.go | 4 + .../config-entries/config_entries_test.go | 5 +- .../external-service.yaml | 2 +- .../kustomization.yaml | 5 + .../terminating-gateway/kustomization.yaml | 5 + .../terminating-gateway.yaml | 0 .../kustomization.yaml | 9 + .../terminating-gateway.yaml | 10 + .../external-service.yaml | 18 + .../kustomization.yaml | 9 + .../kustomization.yaml | 9 + .../external-service.yaml | 3 +- .../kustomization.yaml | 9 + .../terminating-gateway/kustomization.yaml | 9 + .../terminating-gateway.yaml | 0 .../static-client-inject/kustomization.yaml | 9 + .../static-client-inject/patch.yaml | 13 + .../terminating-gateway/kustomization.yaml | 9 + .../terminating-gateway.yaml | 11 + .../tests/peering/peering_connect_test.go | 26 +- .../tests/terminating-gateway/common.go | 46 -- .../terminating_gateway_destinations_test.go | 7 +- .../terminating_gateway_namespaces_test.go | 351 +++++++----- .../terminating_gateway_test.go | 39 +- .../api/v1alpha1/terminatinggateway_types.go | 62 ++- control-plane/catalog/registration/cache.go | 246 ++------ .../registration/registrations_controller.go | 91 +-- .../registrations_controller_test.go | 526 ------------------ control-plane/catalog/registration/result.go | 35 +- control-plane/config/rbac/role.yaml | 20 + .../configentry_controller_test.go | 82 +-- .../terminatinggateway_controller.go | 321 ++++++++++- .../inject-connect/v1controllers.go | 3 +- 38 files changed, 1001 insertions(+), 1163 deletions(-) create mode 100644 .changelog/4224.txt rename acceptance/tests/fixtures/{cases/terminating-gateway => bases/external-service-registration}/external-service.yaml (95%) create mode 100644 acceptance/tests/fixtures/bases/external-service-registration/kustomization.yaml create mode 100644 acceptance/tests/fixtures/bases/terminating-gateway/kustomization.yaml rename acceptance/tests/fixtures/{cases => bases}/terminating-gateway/terminating-gateway.yaml (100%) create mode 100644 acceptance/tests/fixtures/cases/crd-peers/default-terminating-gateway/kustomization.yaml create mode 100644 acceptance/tests/fixtures/cases/crd-peers/default-terminating-gateway/terminating-gateway.yaml create mode 100644 acceptance/tests/fixtures/cases/crd-peers/external-service-registration/external-service.yaml create mode 100644 acceptance/tests/fixtures/cases/crd-peers/external-service-registration/kustomization.yaml create mode 100644 acceptance/tests/fixtures/cases/terminating-gateway-destinations/kustomization.yaml rename acceptance/tests/fixtures/cases/terminating-gateway-namespaces/{ => all-non-default/external-service-registration}/external-service.yaml (91%) create mode 100644 acceptance/tests/fixtures/cases/terminating-gateway-namespaces/all-non-default/external-service-registration/kustomization.yaml create mode 100644 acceptance/tests/fixtures/cases/terminating-gateway-namespaces/all-non-default/terminating-gateway/kustomization.yaml rename acceptance/tests/fixtures/cases/terminating-gateway-namespaces/{ => all-non-default/terminating-gateway}/terminating-gateway.yaml (100%) create mode 100644 acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/static-client-inject/kustomization.yaml create mode 100644 acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/static-client-inject/patch.yaml create mode 100644 acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/terminating-gateway/kustomization.yaml create mode 100644 acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/terminating-gateway/terminating-gateway.yaml diff --git a/.changelog/4224.txt b/.changelog/4224.txt new file mode 100644 index 0000000000..6fde378368 --- /dev/null +++ b/.changelog/4224.txt @@ -0,0 +1,3 @@ +```release-note:bug +terminating-gateways: Fix bug where namespace field was not correctly set on ACL policies if using the `Registration` CRD with the service's namespace unset. +``` diff --git a/acceptance/framework/helpers/helpers.go b/acceptance/framework/helpers/helpers.go index 585eddffd2..b4af57bcea 100644 --- a/acceptance/framework/helpers/helpers.go +++ b/acceptance/framework/helpers/helpers.go @@ -6,10 +6,13 @@ package helpers import ( "context" "encoding/json" + "errors" "fmt" + "net/http" "os" "os/exec" "os/signal" + "slices" "strings" "syscall" "testing" @@ -20,10 +23,12 @@ import ( terratestLogger "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" "github.com/hashicorp/consul-k8s/acceptance/framework/logger" + "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/sdk/testutil" "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -155,22 +160,23 @@ func MergeMaps(a, b map[string]string) { } type K8sOptions struct { - Options *k8s.KubectlOptions - NoCleanupOnFailure bool - NoCleanup bool - ConfigPath string + Options *k8s.KubectlOptions + NoCleanupOnFailure bool + NoCleanup bool + KustomizeConfigPath string } type ConsulOptions struct { - ConsulClient *api.Client - Namespace string + ConsulClient *api.Client + Namespace string + ExternalServiceNameRegistration string } func RegisterExternalServiceCRD(t *testing.T, k8sOptions K8sOptions, consulOptions ConsulOptions) { t.Helper() - t.Logf("Registering external service %s", k8sOptions.ConfigPath) + t.Logf("Registering external service %s", k8sOptions.KustomizeConfigPath) - if consulOptions.Namespace != "" { + if consulOptions.Namespace != "" && consulOptions.Namespace != "default" { logger.Logf(t, "creating the %s namespace in Consul", consulOptions.Namespace) _, _, err := consulOptions.ConsulClient.Namespaces().Create(&api.Namespace{ Name: consulOptions.Namespace, @@ -179,52 +185,34 @@ func RegisterExternalServiceCRD(t *testing.T, k8sOptions K8sOptions, consulOptio } // Register the external service - k8s.KubectlApply(t, k8sOptions.Options, k8sOptions.ConfigPath) - + k8s.KubectlApplyFromKustomize(t, k8sOptions.Options, k8sOptions.KustomizeConfigPath) Cleanup(t, k8sOptions.NoCleanupOnFailure, k8sOptions.NoCleanup, func() { - // Note: this delete command won't wait for pods to be fully terminated. - // This shouldn't cause any test pollution because the underlying - // objects are deployments, and so when other tests create these - // they should have different pod names. - k8s.KubectlDelete(t, k8sOptions.Options, k8sOptions.ConfigPath) + k8s.KubectlDeleteFromKustomize(t, k8sOptions.Options, k8sOptions.KustomizeConfigPath) }) + + CheckExternalServiceConditions(t, consulOptions.ExternalServiceNameRegistration, k8sOptions.Options) } -// RegisterExternalService registers an external service to a virtual node in Consul for testing purposes. -// This function takes a testing.T object, a Consul client, service namespace, service name, address, and port as -// parameters. It registers the service with Consul, and if a namespace is provided, it also creates the namespace -// in Consul. It uses the provided testing.T object to log registration details and verify the registration process. -// If the registration fails, the test calling the function will fail. -// DEPRECATED: Use RegisterExternalServiceCRD instead. -func RegisterExternalService(t *testing.T, consulClient *api.Client, namespace, name, address string, port int) { +func CheckExternalServiceConditions(t *testing.T, registrationName string, opts *k8s.KubectlOptions) { t.Helper() - t.Log("RegisterExternalService is DEPRECATED, use RegisterExternalServiceCRD instead") - - service := &api.AgentService{ - ID: name, - Service: name, - Port: port, - } - - if namespace != "" { - address = fmt.Sprintf("%s.%s", name, namespace) - service.Namespace = namespace - logger.Logf(t, "creating the %s namespace in Consul", namespace) - _, _, err := consulClient.Namespaces().Create(&api.Namespace{ - Name: namespace, - }, nil) - require.NoError(t, err) - } + ogLogger := opts.Logger + defer func() { + opts.Logger = ogLogger + }() - logger.Log(t, fmt.Sprintf("registering the external service %s", name)) - _, err := consulClient.Catalog().Register(&api.CatalogRegistration{ - Node: "external", - Address: address, - NodeMeta: map[string]string{"external-node": "true", "external-probe": "true"}, - Service: service, - }, nil) - require.NoError(t, err) + opts.Logger = terratestLogger.Discard + retry.RunWith(&retry.Counter{Wait: 2 * time.Second, Count: 15}, t, func(r *retry.R) { + var err error + out, err := k8s.RunKubectlAndGetOutputE(r, opts, "get", "-o=json", "registrations.consul.hashicorp.com", registrationName) + require.NoError(r, err) + reg := v1alpha1.Registration{} + err = json.Unmarshal([]byte(out), ®) + require.NoError(r, err) + require.NotEmpty(r, reg.Status.Conditions, "conditions should not be empty, retrying") + // ensure all statuses are true which means that the registration is successful + require.True(r, !slices.ContainsFunc(reg.Status.Conditions, func(c v1alpha1.Condition) bool { return c.Status == corev1.ConditionFalse }), "registration failed because of %v", reg.Status.Conditions) + }) } type Command struct { @@ -358,3 +346,53 @@ func createCmdArgs(options *k8s.KubectlOptions) []string { } return cmdArgs } + +const DEFAULT_PAUSE_PORT = "38501" + +// WaitForInput starts a http server on a random port (which is output in the logs) and waits until you +// issue a request to that endpoint to continue the tests. This is useful for debugging tests that require +// inspecting the current state of a running cluster and you don't need to use long sleeps. +func WaitForInput(t *testing.T) { + t.Helper() + + listenerPort := os.Getenv("CONSUL_K8S_TEST_PAUSE_PORT") + + if listenerPort == "" { + listenerPort = DEFAULT_PAUSE_PORT + } + + mux := http.NewServeMux() + srv := &http.Server{ + Addr: fmt.Sprintf(":%s", listenerPort), + Handler: mux, + } + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + + _, err := w.Write([]byte("input received\n")) + if err != nil { + t.Logf("error writing body: %v", err) + err = nil + } + + err = r.Body.Close() + if err != nil { + t.Logf("error closing request body: %v", err) + err = nil + } + + t.Log("input received, continuing test") + go func() { + err = srv.Shutdown(context.Background()) + if err != nil { + t.Logf("error closing listener: %v", err) + } + }() + }) + + t.Logf("Waiting for input on http://localhost:%s", listenerPort) + if err := srv.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + t.Fatal(err) + } +} diff --git a/acceptance/framework/k8s/deploy.go b/acceptance/framework/k8s/deploy.go index e1d9f01a80..2db7224690 100644 --- a/acceptance/framework/k8s/deploy.go +++ b/acceptance/framework/k8s/deploy.go @@ -165,10 +165,10 @@ func CheckStaticServerConnectionSuccessfulWithMessage(t *testing.T, options *k8s // CheckStaticServerConnectionSuccessful is just like CheckStaticServerConnection // but it always expects a successful connection. -func CheckStaticServerConnectionSuccessful(t *testing.T, options *k8s.KubectlOptions, sourceApp string, curlArgs ...string) { +func CheckStaticServerConnectionSuccessful(t *testing.T, sourceAppOpts *k8s.KubectlOptions, sourceApp string, curlArgs ...string) { t.Helper() start := time.Now() - CheckStaticServerConnection(t, options, sourceApp, true, nil, "", curlArgs...) + CheckStaticServerConnection(t, sourceAppOpts, sourceApp, true, nil, "", curlArgs...) logger.Logf(t, "Took %s to check if static server connection was successful", time.Since(start)) } diff --git a/acceptance/go.mod b/acceptance/go.mod index 2cf54d50bc..04f4b723aa 100644 --- a/acceptance/go.mod +++ b/acceptance/go.mod @@ -5,6 +5,7 @@ go 1.21.1 toolchain go1.22.0 require ( + github.com/go-logr/logr v1.2.4 github.com/google/uuid v1.3.0 github.com/gruntwork-io/terratest v0.46.7 github.com/hashicorp/consul-k8s/control-plane v0.0.0-20240226161840-f3842c41cb2b @@ -30,6 +31,12 @@ require ( sigs.k8s.io/gateway-api v0.7.1 ) +// replace these so we always use the latest version of the control-plane types +replace ( + github.com/hashicorp/consul-k8s/control-plane => ../control-plane + github.com/hashicorp/consul-k8s/version => ../version +) + require ( github.com/armon/go-metrics v0.4.1 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect @@ -45,12 +52,11 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fatih/color v1.16.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect - github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/analysis v0.21.4 // indirect github.com/go-openapi/errors v0.20.3 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect @@ -64,6 +70,7 @@ require ( github.com/go-ozzo/ozzo-validation v3.6.0+incompatible // indirect github.com/go-sql-driver/mysql v1.5.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.5.9 // indirect @@ -71,6 +78,7 @@ require ( github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/gruntwork-io/go-commons v0.8.0 // indirect + github.com/hashicorp/consul-k8s/version v0.0.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-bexpr v0.1.11 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -122,8 +130,6 @@ require ( go.opentelemetry.io/otel/metric v1.19.0 // indirect go.opentelemetry.io/otel/sdk v1.19.0 // indirect go.opentelemetry.io/otel/trace v1.19.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.25.0 // indirect golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect golang.org/x/mod v0.14.0 // indirect @@ -140,6 +146,8 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.28.3 // indirect + k8s.io/component-base v0.28.3 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/acceptance/go.sum b/acceptance/go.sum index de14d4c97c..75e4da2570 100644 --- a/acceptance/go.sum +++ b/acceptance/go.sum @@ -20,9 +20,6 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.44.262 h1:gyXpcJptWoNkK+DiAiaBltlreoWKQXjAIh6FRh60F+I= github.com/aws/aws-sdk-go v1.44.262/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -189,8 +186,6 @@ github.com/gruntwork-io/go-commons v0.8.0 h1:k/yypwrPqSeYHevLlEDmvmgQzcyTwrlZGRa github.com/gruntwork-io/go-commons v0.8.0/go.mod h1:gtp0yTtIBExIZp7vyIV9I0XQkVwiQZze678hvDXof78= github.com/gruntwork-io/terratest v0.46.7 h1:oqGPBBO87SEsvBYaA0R5xOq+Lm2Xc5dmFVfxEolfZeU= github.com/gruntwork-io/terratest v0.46.7/go.mod h1:6gI5MlLeyF+SLwqocA5GBzcTix+XiuxCy1BPwKuT+WM= -github.com/hashicorp/consul-k8s/control-plane v0.0.0-20240226161840-f3842c41cb2b h1:AdeWjUb+rxrRryC5ZHaL32oOZuxubOzV2q6oJ97UMT0= -github.com/hashicorp/consul-k8s/control-plane v0.0.0-20240226161840-f3842c41cb2b/go.mod h1:TVaSJM7vYM/mtKGpVc/Lch53lrqLI9XAXJgy/gY8v4A= github.com/hashicorp/consul-server-connection-manager v0.1.6 h1:ktj8Fi+dRXn9hhM+FXsfEJayhzzgTqfH08Ne5M6Fmug= github.com/hashicorp/consul-server-connection-manager v0.1.6/go.mod h1:HngMIv57MT+pqCVeRQMa1eTB5dqnyMm8uxjyv+Hn8cs= github.com/hashicorp/consul/api v1.29.1 h1:UEwOjYJrd3lG1x5w7HxDRMGiAUPrb3f103EoeKuuEcc= @@ -462,14 +457,10 @@ go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1 go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -486,7 +477,6 @@ golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -560,6 +550,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -589,7 +580,6 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -598,7 +588,6 @@ golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= @@ -654,6 +643,8 @@ k8s.io/apimachinery v0.28.9 h1:aXz4Zxsw+Pk4KhBerAtKRxNN1uSMWKfciL/iOdBfXvA= k8s.io/apimachinery v0.28.9/go.mod h1:zUG757HaKs6Dc3iGtKjzIpBfqTM4yiRsEe3/E7NX15o= k8s.io/client-go v0.28.9 h1:mmMvejwc/KDjMLmDpyaxkWNzlWRCJ6ht7Qsbsnwn39Y= k8s.io/client-go v0.28.9/go.mod h1:GFDy3rUNId++WGrr0hRaBrs+y1eZz5JtVZODEalhRMo= +k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= +k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= diff --git a/acceptance/tests/config-entries/config_entries_namespaces_test.go b/acceptance/tests/config-entries/config_entries_namespaces_test.go index aa74bdc2b5..bcc7605b31 100644 --- a/acceptance/tests/config-entries/config_entries_namespaces_test.go +++ b/acceptance/tests/config-entries/config_entries_namespaces_test.go @@ -90,6 +90,10 @@ func TestControllerNamespaces(t *testing.T) { "global.acls.manageSystemACLs": strconv.FormatBool(c.secure), "global.tls.enabled": strconv.FormatBool(c.secure), + + "terminatingGateways.enabled": "true", + "terminatingGateways.gateways[0].name": "terminating-gateway", + "terminatingGateways.gateways[0].replicas": "1", } releaseName := helpers.RandomName() diff --git a/acceptance/tests/config-entries/config_entries_test.go b/acceptance/tests/config-entries/config_entries_test.go index 9f2595ed4f..2ee242b866 100644 --- a/acceptance/tests/config-entries/config_entries_test.go +++ b/acceptance/tests/config-entries/config_entries_test.go @@ -59,6 +59,10 @@ func TestController(t *testing.T) { "connectInject.enabled": "true", "global.tls.enabled": strconv.FormatBool(c.secure), "global.acls.manageSystemACLs": strconv.FormatBool(c.secure), + + "terminatingGateways.enabled": "true", + "terminatingGateways.gateways[0].name": "terminating-gateway", + "terminatingGateways.gateways[0].replicas": "1", } releaseName := helpers.RandomName() @@ -238,7 +242,6 @@ func TestController(t *testing.T) { require.Equal(r, 100.0, rateLimitIPConfigEntry.Session.WriteRate) require.Equal(r, 100.0, rateLimitIPConfigEntry.Txn.ReadRate) require.Equal(r, 100.0, rateLimitIPConfigEntry.Txn.WriteRate) - }) } diff --git a/acceptance/tests/fixtures/cases/terminating-gateway/external-service.yaml b/acceptance/tests/fixtures/bases/external-service-registration/external-service.yaml similarity index 95% rename from acceptance/tests/fixtures/cases/terminating-gateway/external-service.yaml rename to acceptance/tests/fixtures/bases/external-service-registration/external-service.yaml index 651411f165..156ab3aefb 100644 --- a/acceptance/tests/fixtures/cases/terminating-gateway/external-service.yaml +++ b/acceptance/tests/fixtures/bases/external-service-registration/external-service.yaml @@ -12,4 +12,4 @@ spec: service: id: static-server name: static-server - port: 80 + port: 80 diff --git a/acceptance/tests/fixtures/bases/external-service-registration/kustomization.yaml b/acceptance/tests/fixtures/bases/external-service-registration/kustomization.yaml new file mode 100644 index 0000000000..345a681cd9 --- /dev/null +++ b/acceptance/tests/fixtures/bases/external-service-registration/kustomization.yaml @@ -0,0 +1,5 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - external-service.yaml diff --git a/acceptance/tests/fixtures/bases/terminating-gateway/kustomization.yaml b/acceptance/tests/fixtures/bases/terminating-gateway/kustomization.yaml new file mode 100644 index 0000000000..a5f13bc625 --- /dev/null +++ b/acceptance/tests/fixtures/bases/terminating-gateway/kustomization.yaml @@ -0,0 +1,5 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +resources: + - terminating-gateway.yaml diff --git a/acceptance/tests/fixtures/cases/terminating-gateway/terminating-gateway.yaml b/acceptance/tests/fixtures/bases/terminating-gateway/terminating-gateway.yaml similarity index 100% rename from acceptance/tests/fixtures/cases/terminating-gateway/terminating-gateway.yaml rename to acceptance/tests/fixtures/bases/terminating-gateway/terminating-gateway.yaml diff --git a/acceptance/tests/fixtures/cases/crd-peers/default-terminating-gateway/kustomization.yaml b/acceptance/tests/fixtures/cases/crd-peers/default-terminating-gateway/kustomization.yaml new file mode 100644 index 0000000000..f97dc12cb8 --- /dev/null +++ b/acceptance/tests/fixtures/cases/crd-peers/default-terminating-gateway/kustomization.yaml @@ -0,0 +1,9 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../../../fixtures/bases/terminating-gateway +patches: +- path: terminating-gateway.yaml diff --git a/acceptance/tests/fixtures/cases/crd-peers/default-terminating-gateway/terminating-gateway.yaml b/acceptance/tests/fixtures/cases/crd-peers/default-terminating-gateway/terminating-gateway.yaml new file mode 100644 index 0000000000..74c1a1974c --- /dev/null +++ b/acceptance/tests/fixtures/cases/crd-peers/default-terminating-gateway/terminating-gateway.yaml @@ -0,0 +1,10 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: consul.hashicorp.com/v1alpha1 +kind: TerminatingGateway +metadata: + name: terminating-gateway +spec: + services: + - name: static-server-hostname diff --git a/acceptance/tests/fixtures/cases/crd-peers/external-service-registration/external-service.yaml b/acceptance/tests/fixtures/cases/crd-peers/external-service-registration/external-service.yaml new file mode 100644 index 0000000000..e55f8dd3a7 --- /dev/null +++ b/acceptance/tests/fixtures/cases/crd-peers/external-service-registration/external-service.yaml @@ -0,0 +1,18 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: consul.hashicorp.com/v1alpha1 +kind: Registration +metadata: + name: static-server-registration +spec: + datacenter: server + node: external + nodeMeta: + external-node: "true" + external-probe: "true" + address: static-server.external + service: + id: static-server + name: static-server-hostname + port: 80 diff --git a/acceptance/tests/fixtures/cases/crd-peers/external-service-registration/kustomization.yaml b/acceptance/tests/fixtures/cases/crd-peers/external-service-registration/kustomization.yaml new file mode 100644 index 0000000000..db0a3d9e6c --- /dev/null +++ b/acceptance/tests/fixtures/cases/crd-peers/external-service-registration/kustomization.yaml @@ -0,0 +1,9 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../../bases/external-service-registration +patches: +- path: external-service.yaml diff --git a/acceptance/tests/fixtures/cases/terminating-gateway-destinations/kustomization.yaml b/acceptance/tests/fixtures/cases/terminating-gateway-destinations/kustomization.yaml new file mode 100644 index 0000000000..4c0f462d1f --- /dev/null +++ b/acceptance/tests/fixtures/cases/terminating-gateway-destinations/kustomization.yaml @@ -0,0 +1,9 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../../fixtures/bases/terminating-gateway +patches: +- path: terminating-gateway.yaml diff --git a/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/external-service.yaml b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/all-non-default/external-service-registration/external-service.yaml similarity index 91% rename from acceptance/tests/fixtures/cases/terminating-gateway-namespaces/external-service.yaml rename to acceptance/tests/fixtures/cases/terminating-gateway-namespaces/all-non-default/external-service-registration/external-service.yaml index 5b33ee36e0..b5d3569a9e 100644 --- a/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/external-service.yaml +++ b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/all-non-default/external-service-registration/external-service.yaml @@ -2,7 +2,6 @@ apiVersion: consul.hashicorp.com/v1alpha1 kind: Registration metadata: name: static-server-registration - namespace: ns1 spec: datacenter: dc1 node: external @@ -14,4 +13,4 @@ spec: id: static-server name: static-server namespace: ns1 - port: 80 + port: 80 diff --git a/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/all-non-default/external-service-registration/kustomization.yaml b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/all-non-default/external-service-registration/kustomization.yaml new file mode 100644 index 0000000000..0db7394100 --- /dev/null +++ b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/all-non-default/external-service-registration/kustomization.yaml @@ -0,0 +1,9 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../../../bases/external-service-registration/ +patches: +- path: external-service.yaml diff --git a/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/all-non-default/terminating-gateway/kustomization.yaml b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/all-non-default/terminating-gateway/kustomization.yaml new file mode 100644 index 0000000000..793a233b8f --- /dev/null +++ b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/all-non-default/terminating-gateway/kustomization.yaml @@ -0,0 +1,9 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../../../bases/terminating-gateway +patches: +- path: terminating-gateway.yaml diff --git a/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/terminating-gateway.yaml b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/all-non-default/terminating-gateway/terminating-gateway.yaml similarity index 100% rename from acceptance/tests/fixtures/cases/terminating-gateway-namespaces/terminating-gateway.yaml rename to acceptance/tests/fixtures/cases/terminating-gateway-namespaces/all-non-default/terminating-gateway/terminating-gateway.yaml diff --git a/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/static-client-inject/kustomization.yaml b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/static-client-inject/kustomization.yaml new file mode 100644 index 0000000000..81535d787f --- /dev/null +++ b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/static-client-inject/kustomization.yaml @@ -0,0 +1,9 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../../../bases/static-client/ +patches: +- path: patch.yaml diff --git a/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/static-client-inject/patch.yaml b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/static-client-inject/patch.yaml new file mode 100644 index 0000000000..0879b41557 --- /dev/null +++ b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/static-client-inject/patch.yaml @@ -0,0 +1,13 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-client +spec: + template: + metadata: + annotations: + "consul.hashicorp.com/connect-inject": "true" + "consul.hashicorp.com/connect-service-upstreams": "static-server.default:1234" diff --git a/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/terminating-gateway/kustomization.yaml b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/terminating-gateway/kustomization.yaml new file mode 100644 index 0000000000..793a233b8f --- /dev/null +++ b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/terminating-gateway/kustomization.yaml @@ -0,0 +1,9 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../../../bases/terminating-gateway +patches: +- path: terminating-gateway.yaml diff --git a/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/terminating-gateway/terminating-gateway.yaml b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/terminating-gateway/terminating-gateway.yaml new file mode 100644 index 0000000000..607ed976a7 --- /dev/null +++ b/acceptance/tests/fixtures/cases/terminating-gateway-namespaces/client-non-default/terminating-gateway/terminating-gateway.yaml @@ -0,0 +1,11 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +apiVersion: consul.hashicorp.com/v1alpha1 +kind: TerminatingGateway +metadata: + name: terminating-gateway +spec: + services: + - name: static-server + namespace: default diff --git a/acceptance/tests/peering/peering_connect_test.go b/acceptance/tests/peering/peering_connect_test.go index 6bab0aa909..b2ccf89c63 100644 --- a/acceptance/tests/peering/peering_connect_test.go +++ b/acceptance/tests/peering/peering_connect_test.go @@ -344,19 +344,33 @@ func TestPeering_Connect(t *testing.T) { terminatinggateway.CreateMeshConfigEntry(t, staticClientPeerClient, "") // Create the config entry for the terminating gateway - terminatinggateway.CreateTerminatingGatewayConfigEntry(t, staticServerPeerClient, "", "", externalServerHostnameID) - if c.ACLsEnabled { - // Allow the terminating gateway write access to services prefixed with "static-server". - terminatinggateway.UpdateTerminatingGatewayRole(t, staticServerPeerClient, terminatingGatewayRules) - } + logger.Log(t, "creating terminating gateway") + k8s.KubectlApplyK(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/cases/crd-peers/default-terminating-gateway") + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + k8s.KubectlDeleteK(t, staticServerPeerClusterContext.KubectlOptions(t), "../fixtures/cases/crd-peers/default-terminating-gateway") + }) // This is the URL that the static-client will use to dial the external static server in the server peer. externalServerHostnameURL := fmt.Sprintf("http://%s.virtual.%s.consul", externalServerHostnameID, staticServerPeer) // Register the external service. terminatinggateway.CreateServiceDefaultDestination(t, staticServerPeerClient, "", externalServerHostnameID, "http", 80, fmt.Sprintf("%s.%s", externalServerServiceName, externalServerK8sNamespace)) + + // Register the external service + k8sOptions := helpers.K8sOptions{ + Options: staticServerPeerClusterContext.KubectlOptions(t), + NoCleanupOnFailure: cfg.NoCleanupOnFailure, + NoCleanup: cfg.NoCleanup, + KustomizeConfigPath: "../fixtures/cases/crd-peers/external-service-registration", + } + + consulOptions := helpers.ConsulOptions{ + ConsulClient: staticServerPeerClient, + ExternalServiceNameRegistration: "static-server-registration", + } + // (t-eckert) this shouldn't be required but currently is with HTTP services. It works around a bug. - helpers.RegisterExternalService(t, staticServerPeerClient, "", externalServerHostnameID, fmt.Sprintf("%s.%s", externalServerServiceName, externalServerK8sNamespace), 80) + helpers.RegisterExternalServiceCRD(t, k8sOptions, consulOptions) // Export the external service to the client peer. logger.Log(t, "creating exported external services") diff --git a/acceptance/tests/terminating-gateway/common.go b/acceptance/tests/terminating-gateway/common.go index 36b5293c2b..d02bde16a7 100644 --- a/acceptance/tests/terminating-gateway/common.go +++ b/acceptance/tests/terminating-gateway/common.go @@ -8,8 +8,6 @@ import ( "strings" "testing" - "github.com/gruntwork-io/terratest/modules/k8s" - "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" "github.com/hashicorp/consul-k8s/acceptance/framework/logger" "github.com/hashicorp/consul/api" "github.com/stretchr/testify/require" @@ -41,50 +39,6 @@ func AddIntention(t *testing.T, consulClient *api.Client, sourcePeer, sourceNS, require.NoError(t, err) } -func CreateTerminatingGatewayFromCRD(t *testing.T, kubectlOptions *k8s.KubectlOptions, noCleanupOnFailure, noCleanup bool, path string) { - // Create the config entry for the terminating gateway. - k8s.KubectlApply(t, kubectlOptions, path) - - helpers.Cleanup(t, noCleanupOnFailure, noCleanup, func() { - // Note: this delete command won't wait for pods to be fully terminated. - // This shouldn't cause any test pollution because the underlying - // objects are deployments, and so when other tests create these - // they should have different pod names. - k8s.KubectlDelete(t, kubectlOptions, path) - }) -} - -func CreateTerminatingGatewayConfigEntry(t *testing.T, consulClient *api.Client, gwNamespace, serviceNamespace string, serviceNames ...string) { - t.Helper() - - logger.Log(t, "creating config entry") - - if serviceNamespace != "" { - logger.Logf(t, "creating the %s namespace in Consul", serviceNamespace) - _, _, err := consulClient.Namespaces().Create(&api.Namespace{ - Name: serviceNamespace, - }, nil) - require.NoError(t, err) - } - - var gatewayServices []api.LinkedService - for _, serviceName := range serviceNames { - linkedService := api.LinkedService{Name: serviceName, Namespace: serviceNamespace} - gatewayServices = append(gatewayServices, linkedService) - } - - configEntry := &api.TerminatingGatewayConfigEntry{ - Kind: api.TerminatingGateway, - Name: "terminating-gateway", - Namespace: gwNamespace, - Services: gatewayServices, - } - - created, _, err := consulClient.ConfigEntries().Set(configEntry, nil) - require.NoError(t, err) - require.True(t, created, "failed to create config entry") -} - func UpdateTerminatingGatewayRole(t *testing.T, consulClient *api.Client, rules string) { t.Helper() diff --git a/acceptance/tests/terminating-gateway/terminating_gateway_destinations_test.go b/acceptance/tests/terminating-gateway/terminating_gateway_destinations_test.go index 753e49580a..3edb4a3b71 100644 --- a/acceptance/tests/terminating-gateway/terminating_gateway_destinations_test.go +++ b/acceptance/tests/terminating-gateway/terminating_gateway_destinations_test.go @@ -78,6 +78,7 @@ func TestTerminatingGatewayDestinations(t *testing.T) { // with service:write permissions to the static-server service // so that it can request Connect certificates for it. if c.secure { + logger.Log(t, "updating acl role") UpdateTerminatingGatewayRole(t, consulClient, terminatingGatewayRules) } @@ -86,7 +87,11 @@ func TestTerminatingGatewayDestinations(t *testing.T) { CreateMeshConfigEntry(t, consulClient, "") // Create the config entry for the terminating gateway. - CreateTerminatingGatewayFromCRD(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, "../fixtures/cases/terminating-gateway-destinations/terminating-gateway.yaml") + logger.Log(t, "creating terminating gateway") + k8s.KubectlApplyK(t, ctx.KubectlOptions(t), "../fixtures/cases/terminating-gateway-destinations") + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + k8s.KubectlDeleteK(t, ctx.KubectlOptions(t), "../fixtures/cases/terminating-gateway-destinations") + }) // Deploy the static client logger.Log(t, "deploying static client") diff --git a/acceptance/tests/terminating-gateway/terminating_gateway_namespaces_test.go b/acceptance/tests/terminating-gateway/terminating_gateway_namespaces_test.go index 6407a483ae..2bec2b1698 100644 --- a/acceptance/tests/terminating-gateway/terminating_gateway_namespaces_test.go +++ b/acceptance/tests/terminating-gateway/terminating_gateway_namespaces_test.go @@ -78,28 +78,25 @@ func TestTerminatingGatewaySingleNamespace(t *testing.T) { // Register the external service k8sOptions := helpers.K8sOptions{ - Options: ctx.KubectlOptions(t), - NoCleanupOnFailure: cfg.NoCleanupOnFailure, - NoCleanup: cfg.NoCleanup, - ConfigPath: "../fixtures/cases/terminating-gateway-namespaces/external-service.yaml", + Options: ctx.KubectlOptions(t), + NoCleanupOnFailure: cfg.NoCleanupOnFailure, + NoCleanup: cfg.NoCleanup, + KustomizeConfigPath: "../fixtures/cases/terminating-gateway-namespaces/all-non-default/external-service-registration/", } consulOptions := helpers.ConsulOptions{ - ConsulClient: consulClient, + ConsulClient: consulClient, + Namespace: testNamespace, + ExternalServiceNameRegistration: "static-server-registration", } helpers.RegisterExternalServiceCRD(t, k8sOptions, consulOptions) - // If ACLs are enabled we need to update the role of the terminating gateway - // with service:write permissions to the static-server service - // so that it can request Connect certificates for it. - if c.secure { - UpdateTerminatingGatewayRole(t, consulClient, fmt.Sprintf(staticServerPolicyRulesNamespace, testNamespace)) - } - - // Create the config entry for the terminating gateway. - // This case cannot be replicated using CRDs because the consul namespace does not match the kubernetes namespace the terminating gateway is in - CreateTerminatingGatewayConfigEntry(t, consulClient, testNamespace, testNamespace, staticServerName) + logger.Log(t, "creating terminating gateway") + k8s.KubectlApplyK(t, nsK8SOptions, "../fixtures/cases/terminating-gateway-namespaces/all-non-default/terminating-gateway") + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + k8s.KubectlDeleteK(t, nsK8SOptions, "../fixtures/cases/terminating-gateway-namespaces/all-non-default/terminating-gateway") + }) // Deploy the static client. logger.Log(t, "deploying static client") @@ -125,127 +122,229 @@ func TestTerminatingGatewaySingleNamespace(t *testing.T) { } // Test we can connect through the terminating gateway when the terminating gateway, -// the external service, and the connect service are in different namespace. +// the external service, and the connect service are in different combinations of namespaces. func TestTerminatingGatewayNamespaceMirroring(t *testing.T) { cfg := suite.Config() if !cfg.EnableEnterprise { t.Skipf("skipping this test because -enable-enterprise is not set") } - cases := []struct { - secure bool + type config struct { + path string + namespace string + } + + // for simplicity/to keep from an explosion of test cases we're keeping the registration in the same namespace as the + // service being registered, this shouldn't matter because external services should be outside of the cluster typically + cases := map[string]struct { + termGWConfig config + externalServiceRegistrationConfig config + staticServerConfig config + staticClientConfig config }{ - { - secure: false, + "all in default namespace": { + termGWConfig: config{ + path: "../fixtures/bases/terminating-gateway", + namespace: "default", + }, + externalServiceRegistrationConfig: config{ + path: "../fixtures/bases/external-service-registration", + namespace: "default", + }, + staticServerConfig: config{ + path: "../fixtures/bases/static-server", + namespace: "default", + }, + staticClientConfig: config{ + path: "../fixtures/cases/static-client-inject", + namespace: "default", + }, }, - { - secure: true, + "all in same non-default namespace": { + termGWConfig: config{ + path: "../fixtures/cases/terminating-gateway-namespaces/all-non-default/terminating-gateway", + namespace: "ns1", + }, + externalServiceRegistrationConfig: config{ + path: "../fixtures/cases/terminating-gateway-namespaces/all-non-default/external-service-registration", + namespace: "ns1", + }, + staticServerConfig: config{ + path: "../fixtures/bases/static-server", + namespace: "ns1", + }, + staticClientConfig: config{ + path: "../fixtures/cases/static-client-namespaces", + namespace: "ns1", + }, + }, + "mesh service in default namespace everything else in non-default namespace": { + termGWConfig: config{ + path: "../fixtures/cases/terminating-gateway-namespaces/all-non-default/terminating-gateway", + namespace: "ns1", + }, + externalServiceRegistrationConfig: config{ + path: "../fixtures/cases/terminating-gateway-namespaces/all-non-default/external-service-registration", + namespace: "ns1", + }, + staticServerConfig: config{ + path: "../fixtures/bases/static-server", + namespace: "ns1", + }, + staticClientConfig: config{ + path: "../fixtures/cases/static-client-namespaces", + namespace: "default", + }, }, + "external service in default namespace everything else in non-default namespace": { + termGWConfig: config{ + path: "../fixtures/cases/terminating-gateway-namespaces/client-non-default/terminating-gateway", + namespace: "ns1", + }, + externalServiceRegistrationConfig: config{ + path: "../fixtures/bases/external-service-registration", + namespace: "default", + }, + staticServerConfig: config{ + path: "../fixtures/bases/static-server", + namespace: "default", + }, + staticClientConfig: config{ + path: "../fixtures/cases/terminating-gateway-namespaces/client-non-default/static-client-inject", + namespace: "ns1", + }, + }, + // TODO: (NET-10248) need to dig in more on why this isn't working when acls are enabled. + // "terminating gateway in default namespace everything else in non-default namespace": { + // termGWConfig: config{ + // path: "../fixtures/cases/terminating-gateway-namespaces/all-non-default/terminating-gateway", + // namespace: "default", + // }, + // externalServiceRegistrationConfig: config{ + // path: "../fixtures/cases/terminating-gateway-namespaces/all-non-default/external-service-registration", + // namespace: "ns1", + // }, + // staticServerConfig: config{ + // path: "../fixtures/bases/static-server", + // namespace: "ns1", + // }, + // staticClientConfig: config{ + // path: "../fixtures/cases/static-client-namespaces", + // namespace: "ns1", + // }, + // }, } - for _, c := range cases { - name := fmt.Sprintf("secure: %t", c.secure) - t.Run(name, func(t *testing.T) { - ctx := suite.Environment().DefaultContext(t) - - // Install the Helm chart without the terminating gateway first - // so that we can create the namespace for it. - helmValues := map[string]string{ - "connectInject.enabled": "true", - "connectInject.consulNamespaces.mirroringK8S": "true", - - "global.enableConsulNamespaces": "true", - "global.acls.manageSystemACLs": strconv.FormatBool(c.secure), - "global.tls.enabled": strconv.FormatBool(c.secure), - - "terminatingGateways.enabled": "true", - "terminatingGateways.gateways[0].name": "terminating-gateway", - "terminatingGateways.gateways[0].replicas": "1", - } - - releaseName := helpers.RandomName() - consulCluster := consul.NewHelmCluster(t, helmValues, ctx, cfg, releaseName) - - consulCluster.Create(t) - - consulClient, _ := consulCluster.SetupConsulClient(t, c.secure) - - logger.Logf(t, "creating Kubernetes namespace %s", testNamespace) - k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", testNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { - k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", testNamespace) - }) - - StaticClientNamespace := "ns2" - logger.Logf(t, "creating Kubernetes namespace %s", StaticClientNamespace) - k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", StaticClientNamespace) - helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { - k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", StaticClientNamespace) + for name, tc := range cases { + for _, secure := range []bool{true, false} { + name := fmt.Sprintf("%s secure: %t", name, secure) + t.Run(name, func(t *testing.T) { + ctx := suite.Environment().DefaultContext(t) + + // Install the Helm chart without the terminating gateway first + // so that we can create the namespace for it. + helmValues := map[string]string{ + "connectInject.enabled": "true", + "connectInject.consulNamespaces.mirroringK8S": "true", + + "global.enableConsulNamespaces": "true", + "global.acls.manageSystemACLs": strconv.FormatBool(secure), + "global.tls.enabled": strconv.FormatBool(secure), + + "terminatingGateways.enabled": "true", + "terminatingGateways.gateways[0].name": "terminating-gateway", + "terminatingGateways.gateways[0].replicas": "1", + "terminatingGateways.gateways[0].consulNamespace": tc.termGWConfig.namespace, + } + + releaseName := helpers.RandomName() + consulCluster := consul.NewHelmCluster(t, helmValues, ctx, cfg, releaseName) + + consulCluster.Create(t) + + consulClient, _ := consulCluster.SetupConsulClient(t, secure) + + seen := make(map[string]struct{}, 4) + for _, ns := range []string{tc.externalServiceRegistrationConfig.namespace, tc.staticServerConfig.namespace, tc.staticClientConfig.namespace, tc.termGWConfig.namespace} { + _, ok := seen[ns] + if ns != "default" && !ok { + logger.Logf(t, "creating Kubernetes namespace %s", ns) + k8s.RunKubectl(t, ctx.KubectlOptions(t), "create", "ns", ns) + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + k8s.RunKubectl(t, ctx.KubectlOptions(t), "delete", "ns", ns) + }) + seen[ns] = struct{}{} + } + } + + staticServerNSOpts := &terratestk8s.KubectlOptions{ + ContextName: ctx.KubectlOptions(t).ContextName, + ConfigPath: ctx.KubectlOptions(t).ConfigPath, + Namespace: tc.staticServerConfig.namespace, + } + + staticClientNSOpts := &terratestk8s.KubectlOptions{ + ContextName: ctx.KubectlOptions(t).ContextName, + ConfigPath: ctx.KubectlOptions(t).ConfigPath, + Namespace: tc.staticClientConfig.namespace, + } + + termGWNSOpts := &terratestk8s.KubectlOptions{ + ContextName: ctx.KubectlOptions(t).ContextName, + ConfigPath: ctx.KubectlOptions(t).ConfigPath, + Namespace: tc.termGWConfig.namespace, + } + + externalServiceRegistrationNSOpts := &terratestk8s.KubectlOptions{ + ContextName: ctx.KubectlOptions(t).ContextName, + ConfigPath: ctx.KubectlOptions(t).ConfigPath, + Namespace: tc.externalServiceRegistrationConfig.namespace, + } + + // Deploy a static-server that will play the role of an external service. + logger.Log(t, "creating static-server deployment") + k8s.DeployKustomize(t, staticServerNSOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, tc.staticServerConfig.path) + + // Create the config entry for the terminating gateway. + logger.Log(t, "creating terminating gateway") + k8s.KubectlApplyK(t, termGWNSOpts, tc.termGWConfig.path) + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + k8s.KubectlDeleteK(t, termGWNSOpts, tc.termGWConfig.path) + }) + + k8sOpts := helpers.K8sOptions{ + Options: externalServiceRegistrationNSOpts, + NoCleanupOnFailure: cfg.NoCleanupOnFailure, + NoCleanup: cfg.NoCleanup, + KustomizeConfigPath: tc.externalServiceRegistrationConfig.path, + } + + consulOpts := helpers.ConsulOptions{ + ConsulClient: consulClient, + Namespace: tc.externalServiceRegistrationConfig.namespace, + ExternalServiceNameRegistration: "static-server-registration", + } + + helpers.RegisterExternalServiceCRD(t, k8sOpts, consulOpts) + + // Deploy the static client + logger.Log(t, "deploying static client") + k8s.DeployKustomize(t, staticClientNSOpts, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, tc.staticClientConfig.path) + // If ACLs are enabled, test that intentions prevent connections. + if secure { + // With the terminating gateway up, we test that we can make a call to it + // via the static-server. It should fail to connect with the + // static-server pod because of intentions. + logger.Log(t, "testing intentions prevent connections through the terminating gateway") + k8s.CheckStaticServerConnectionFailing(t, staticClientNSOpts, staticClientName, staticServerLocalAddress) + + logger.Log(t, "adding intentions to allow traffic from client ==> server") + AddIntention(t, consulClient, "", tc.staticClientConfig.namespace, staticClientName, tc.staticServerConfig.namespace, staticServerName) + } + + // Test that we can make a call to the terminating gateway + logger.Log(t, "trying calls to terminating gateway") + k8s.CheckStaticServerConnectionSuccessful(t, staticClientNSOpts, staticClientName, staticServerLocalAddress) }) - - ns1K8SOptions := &terratestk8s.KubectlOptions{ - ContextName: ctx.KubectlOptions(t).ContextName, - ConfigPath: ctx.KubectlOptions(t).ConfigPath, - Namespace: testNamespace, - } - ns2K8SOptions := &terratestk8s.KubectlOptions{ - ContextName: ctx.KubectlOptions(t).ContextName, - ConfigPath: ctx.KubectlOptions(t).ConfigPath, - Namespace: StaticClientNamespace, - } - - // Deploy a static-server that will play the role of an external service. - logger.Log(t, "creating static-server deployment") - k8s.DeployKustomize(t, ns1K8SOptions, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/bases/static-server") - - // Register the external service - k8sOptions := helpers.K8sOptions{ - Options: ctx.KubectlOptions(t), - NoCleanupOnFailure: cfg.NoCleanupOnFailure, - NoCleanup: cfg.NoCleanup, - ConfigPath: "../fixtures/cases/terminating-gateway-namespaces/external-service.yaml", - } - - consulOptions := helpers.ConsulOptions{ - ConsulClient: consulClient, - Namespace: testNamespace, - } - - helpers.RegisterExternalServiceCRD(t, k8sOptions, consulOptions) - - // If ACLs are enabled we need to update the role of the terminating gateway - // with service:write permissions to the static-server service - // so that it can request Connect certificates for it. - if c.secure { - UpdateTerminatingGatewayRole(t, consulClient, fmt.Sprintf(staticServerPolicyRulesNamespace, testNamespace)) - } - - // Create the config entry for the terminating gateway. - CreateTerminatingGatewayFromCRD(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, "../fixtures/cases/terminating-gateway-namespaces/terminating-gateway.yaml") - - // Deploy the static client - logger.Log(t, "deploying static client") - k8s.DeployKustomize(t, ns2K8SOptions, cfg.NoCleanupOnFailure, cfg.NoCleanup, cfg.DebugDirectory, "../fixtures/cases/static-client-namespaces") - - // If ACLs are enabled, test that intentions prevent connections. - if c.secure { - // With the terminating gateway up, we test that we can make a call to it - // via the static-server. It should fail to connect with the - // static-server pod because of intentions. - logger.Log(t, "testing intentions prevent connections through the terminating gateway") - k8s.CheckStaticServerConnectionFailing(t, ns2K8SOptions, staticClientName, staticServerLocalAddress) - - logger.Log(t, "adding intentions to allow traffic from client ==> server") - AddIntention(t, consulClient, "", StaticClientNamespace, staticClientName, testNamespace, staticServerName) - } - - // Test that we can make a call to the terminating gateway - logger.Log(t, "trying calls to terminating gateway") - k8s.CheckStaticServerConnectionSuccessful(t, ns2K8SOptions, staticClientName, staticServerLocalAddress) - }) + } } } - -const staticServerPolicyRulesNamespace = `namespace %q { -service "static-server" { - policy = "write" -}}` diff --git a/acceptance/tests/terminating-gateway/terminating_gateway_test.go b/acceptance/tests/terminating-gateway/terminating_gateway_test.go index bf3cd44d60..168fa497a0 100644 --- a/acceptance/tests/terminating-gateway/terminating_gateway_test.go +++ b/acceptance/tests/terminating-gateway/terminating_gateway_test.go @@ -54,30 +54,27 @@ func TestTerminatingGateway(t *testing.T) { // Once the cluster is up, register the external service, then create the config entry. consulClient, _ := consulCluster.SetupConsulClient(t, c.secure) - // Register the external service - k8sOptions := helpers.K8sOptions{ - Options: ctx.KubectlOptions(t), - NoCleanupOnFailure: cfg.NoCleanupOnFailure, - NoCleanup: cfg.NoCleanup, - ConfigPath: "../fixtures/cases/terminating-gateway/external-service.yaml", + logger.Log(t, "creating terminating gateway") + k8s.KubectlApplyK(t, ctx.KubectlOptions(t), "../fixtures/bases/terminating-gateway") + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + k8s.KubectlDeleteK(t, ctx.KubectlOptions(t), "../fixtures/bases/terminating-gateway") + }) + + k8sOpts := helpers.K8sOptions{ + Options: ctx.KubectlOptions(t), + NoCleanupOnFailure: cfg.NoCleanupOnFailure, + NoCleanup: cfg.NoCleanup, + KustomizeConfigPath: "../fixtures/bases/external-service-registration", } - consulOptions := helpers.ConsulOptions{ - ConsulClient: consulClient, + consulOpts := helpers.ConsulOptions{ + ConsulClient: consulClient, + ExternalServiceNameRegistration: "static-server-registration", } - helpers.RegisterExternalServiceCRD(t, k8sOptions, consulOptions) + helpers.RegisterExternalServiceCRD(t, k8sOpts, consulOpts) - // If ACLs are enabled we need to update the role of the terminating gateway - // with service:write permissions to the static-server service - // so that it can request Connect certificates for it. - if c.secure { - UpdateTerminatingGatewayRole(t, consulClient, staticServerPolicyRules) - } - - logger.Log(t, "creating terminating gateway config entry") - // Create the config entry for the terminating gateway. - CreateTerminatingGatewayFromCRD(t, ctx.KubectlOptions(t), cfg.NoCleanupOnFailure, cfg.NoCleanup, "../fixtures/cases/terminating-gateway/terminating-gateway.yaml") + helpers.CheckExternalServiceConditions(t, "static-server-registration", k8sOpts.Options) // Deploy the static client logger.Log(t, "deploying static client") @@ -101,7 +98,3 @@ func TestTerminatingGateway(t *testing.T) { }) } } - -const staticServerPolicyRules = `service "static-server" { - policy = "write" -}` diff --git a/control-plane/api/v1alpha1/terminatinggateway_types.go b/control-plane/api/v1alpha1/terminatinggateway_types.go index d439e635fe..ca2d688011 100644 --- a/control-plane/api/v1alpha1/terminatinggateway_types.go +++ b/control-plane/api/v1alpha1/terminatinggateway_types.go @@ -22,6 +22,13 @@ const ( terminatingGatewayKubeKind = "terminatinggateway" ) +const ( + TerminatingGatewayFailedToSetACLs string = "FailedToSetACLs" +) + +// Condition Type. +const ConsulACLStatus ConditionType = "ConsulACLsSynced" + func init() { SchemeBuilder.Register(&TerminatingGateway{}, &TerminatingGatewayList{}) } @@ -80,10 +87,21 @@ type LinkedService struct { // SNI is the optional name to specify during the TLS handshake with a linked service. SNI string `json:"sni,omitempty"` - //DisableAutoHostRewrite disables terminating gateways auto host rewrite feature when set to true. + // DisableAutoHostRewrite disables terminating gateways auto host rewrite feature when set to true. DisableAutoHostRewrite bool `json:"disableAutoHostRewrite,omitempty"` } +func (l LinkedService) NamespaceName() string { + return defaultIfEmpty(l.Namespace) + "." + l.Name +} + +func defaultIfEmpty(s string) string { + if s == "" { + return "default" + } + return s +} + func (in *TerminatingGateway) GetObjectMeta() metav1.ObjectMeta { return in.ObjectMeta } @@ -131,15 +149,41 @@ func (in *TerminatingGateway) KubernetesName() string { } func (in *TerminatingGateway) SetSyncedCondition(status corev1.ConditionStatus, reason, message string) { - in.Status.Conditions = Conditions{ - { - Type: ConditionSynced, - Status: status, - LastTransitionTime: metav1.Now(), - Reason: reason, - Message: message, - }, + cond := Condition{ + Type: ConditionSynced, + Status: status, + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: message, + } + + for idx, c := range in.Status.Conditions { + if c.Type == ConditionSynced { + in.Status.Conditions[idx] = cond + return + } + } + + in.Status.Conditions = append(in.Status.Conditions, cond) +} + +func (in *TerminatingGateway) SetACLStatusConditon(status corev1.ConditionStatus, reason, message string) { + cond := Condition{ + Type: ConsulACLStatus, + Status: status, + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: message, } + + for idx, c := range in.Status.Conditions { + if c.Type == ConsulACLStatus { + in.Status.Conditions[idx] = cond + return + } + } + + in.Status.Conditions = append(in.Status.Conditions, cond) } func (in *TerminatingGateway) SetLastSyncedTime(time *metav1.Time) { diff --git a/control-plane/catalog/registration/cache.go b/control-plane/catalog/registration/cache.go index 357a59b33c..9fce1dde54 100644 --- a/control-plane/catalog/registration/cache.go +++ b/control-plane/catalog/registration/cache.go @@ -4,60 +4,20 @@ package registration import ( - "bytes" "context" - "errors" - "fmt" - "slices" "strings" "sync" - "text/template" mapset "github.com/deckarep/golang-set/v2" "github.com/go-logr/logr" "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" "github.com/hashicorp/consul-k8s/control-plane/consul" capi "github.com/hashicorp/consul/api" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) const NotInServiceMeshFilter = "ServiceMeta[\"managed-by\"] != \"consul-k8s-endpoints-controller\"" -func init() { - gatewayTpl = template.Must(template.New("root").Parse(strings.TrimSpace(gatewayRulesTpl))) -} - -type templateArgs struct { - EnablePartitions bool - Partition string - EnableNamespaces bool - Namespace string - ServiceName string -} - -var ( - gatewayTpl *template.Template - gatewayRulesTpl = ` -{{ if .EnablePartitions }} -partition "{{.Partition}}" { -{{- end }} - {{- if .EnableNamespaces }} - namespace "{{.Namespace}}" { - {{- end }} - service "{{.ServiceName}}" { - policy = "write" - } - {{- if .EnableNamespaces }} - } - {{- end }} -{{- if .EnablePartitions }} -} -{{- end }} -` -) - type RegistrationCache struct { // we include the context here so that we can use it for cancellation of `run` invocations that are scheduled after the cache is started // this occurs when registering services in a new namespace as we have an invocation of `run` per namespace that is registered @@ -115,12 +75,12 @@ func (c *RegistrationCache) run(log logr.Logger, namespace string) { return default: - client, err := consul.NewClientFromConnMgr(c.ConsulClientConfig, c.ConsulServerConnMgr) + consulClient, err := consul.NewClientFromConnMgr(c.ConsulClientConfig, c.ConsulServerConnMgr) if err != nil { log.Error(err, "error initializing consul client") continue } - entries, meta, err := client.Catalog().Services(opts.WithContext(c.ctx)) + entries, meta, err := consulClient.Catalog().Services(opts.WithContext(c.ctx)) if err != nil { // if we timeout we don't care about the error message because it's expected to happen on long polls // any other error we want to alert on @@ -135,14 +95,17 @@ func (c *RegistrationCache) run(log logr.Logger, namespace string) { servicesToRemove := mapset.NewSet[string]() servicesToAdd := mapset.NewSet[string]() c.serviceMtx.Lock() + + // Remove any services in the cache that are no longer in consul for svc := range c.Services { if _, ok := entries[svc]; !ok { servicesToRemove.Add(svc) } } + // Add any services to the cache that are in consul but not in the cache (we expect to hit this loop on a reboot) for svc := range entries { - if _, ok := c.Services[svc]; !ok { + if _, ok := c.Services[svc]; !ok && svc != "consul" { servicesToAdd.Add(svc) } } @@ -154,18 +117,28 @@ func (c *RegistrationCache) run(log logr.Logger, namespace string) { } for _, svc := range servicesToAdd.ToSlice() { - registration := &v1alpha1.Registration{} + log.Info("consul registered service", "svcName", svc) + registrationList := &v1alpha1.RegistrationList{} + + if err := c.k8sClient.List(context.Background(), registrationList, client.MatchingFields{registrationByServiceNameIndex: svc}); err != nil { + log.Error(err, "error listing registrations", "svcName", svc) + } - if err := c.k8sClient.Get(c.ctx, types.NamespacedName{Name: svc, Namespace: namespace}, registration); err != nil { - if !k8serrors.IsNotFound(err) { - log.Error(err, "unable to get registration", "svcName", svc, "namespace", namespace) + found := false + for _, reg := range registrationList.Items { + if reg.Spec.Service.Name == svc { + found = true + c.set(svc, ®) } - continue } - c.Services[svc] = registration + if !found { + log.Info("registration not found in k8s", "svcName", svc) + } } + log.Info("synced registrations with consul") + opts.WaitIndex = meta.LastIndex once.Do(func() { log.Info("Initial sync complete") @@ -182,11 +155,20 @@ func (c *RegistrationCache) get(svcName string) (*v1alpha1.Registration, bool) { return val, ok } -func (c *RegistrationCache) aclsEnabled() bool { - return c.ConsulClientConfig.APIClientConfig.Token != "" || c.ConsulClientConfig.APIClientConfig.TokenFile != "" +func (c *RegistrationCache) set(name string, reg *v1alpha1.Registration) { + c.serviceMtx.Lock() + defer c.serviceMtx.Unlock() + c.Services[name] = reg } func (c *RegistrationCache) registerService(log logr.Logger, reg *v1alpha1.Registration) error { + if svc, ok := c.get(reg.Spec.Service.Name); ok { + if reg.EqualExceptStatus(svc) { + log.Info("service already registered", "svcName", reg.Spec.Service.Name) + return nil + } + } + client, err := consul.NewClientFromConnMgr(c.ConsulClientConfig, c.ConsulServerConnMgr) if err != nil { return err @@ -213,94 +195,6 @@ func (c *RegistrationCache) registerService(log logr.Logger, reg *v1alpha1.Regis return nil } -func emptyOrDefault(s string) bool { - return s == "" || s == "default" -} - -func (c *RegistrationCache) updateTermGWACLRole(log logr.Logger, registration *v1alpha1.Registration, termGWsToUpdate []v1alpha1.TerminatingGateway) error { - if len(termGWsToUpdate) == 0 { - log.Info("terminating gateway not found") - return nil - } - - client, err := consul.NewClientFromConnMgr(c.ConsulClientConfig, c.ConsulServerConnMgr) - if err != nil { - return err - } - - var data bytes.Buffer - if err := gatewayTpl.Execute(&data, templateArgs{ - EnablePartitions: c.partitionsEnabled, - Partition: registration.Spec.Service.Partition, - EnableNamespaces: c.namespacesEnabled, - Namespace: registration.Spec.Service.Namespace, - ServiceName: registration.Spec.Service.Name, - }); err != nil { - // just panic if we can't compile the simple template - // as it means something else is going severly wrong. - panic(err) - } - - var mErr error - for _, termGW := range termGWsToUpdate { - // the terminating gateway role is _always_ in the default namespace - roles, _, err := client.ACL().RoleList(&capi.QueryOptions{}) - if err != nil { - log.Error(err, "error reading role list") - return err - } - - policy := &capi.ACLPolicy{ - Name: servicePolicyName(registration.Spec.Service.Name), - Description: "Write policy for terminating gateways for external service", - Rules: data.String(), - Datacenters: []string{registration.Spec.Datacenter}, - } - - existingPolicy, _, err := client.ACL().PolicyReadByName(policy.Name, &capi.QueryOptions{}) - if err != nil { - log.Error(err, "error reading policy") - return err - } - - // we don't need to include the namespace/partition here because all roles and policies are created in the default namespace for consul-k8s managed resources. - writeOpts := &capi.WriteOptions{} - - if existingPolicy == nil { - policy, _, err = client.ACL().PolicyCreate(policy, writeOpts) - if err != nil { - return fmt.Errorf("error creating policy: %w", err) - } - } else { - policy = existingPolicy - } - var role *capi.ACLRole - for _, r := range roles { - if strings.HasSuffix(r.Name, fmt.Sprintf("-%s-acl-role", termGW.Name)) { - role = r - break - } - } - - if role == nil { - log.Info("terminating gateway role not found", "terminatingGatewayName", termGW.Name) - mErr = errors.Join(mErr, fmt.Errorf("terminating gateway role not found for %q", termGW.Name)) - continue - } - - role.Policies = append(role.Policies, &capi.ACLRolePolicyLink{Name: policy.Name, ID: policy.ID}) - - _, _, err = client.ACL().RoleUpdate(role, writeOpts) - if err != nil { - log.Error(err, "error updating role", "roleName", role.Name) - mErr = errors.Join(mErr, fmt.Errorf("error updating role %q", role.Name)) - continue - } - } - - return mErr -} - func (c *RegistrationCache) deregisterService(log logr.Logger, reg *v1alpha1.Registration) error { client, err := consul.NewClientFromConnMgr(c.ConsulClientConfig, c.ConsulServerConnMgr) if err != nil { @@ -322,76 +216,6 @@ func (c *RegistrationCache) deregisterService(log logr.Logger, reg *v1alpha1.Reg return nil } -func (c *RegistrationCache) removeTermGWACLRole(log logr.Logger, registration *v1alpha1.Registration, termGWsToUpdate []v1alpha1.TerminatingGateway) error { - if len(termGWsToUpdate) == 0 { - log.Info("terminating gateway not found") - return nil - } - - client, err := consul.NewClientFromConnMgr(c.ConsulClientConfig, c.ConsulServerConnMgr) - if err != nil { - return err - } - - var mErr error - for _, termGW := range termGWsToUpdate { - - // we don't need to include the namespace/partition here because all roles and policies are created in the default namespace for consul-k8s managed resources. - queryOpts := &capi.QueryOptions{} - writeOpts := &capi.WriteOptions{} - - roles, _, err := client.ACL().RoleList(queryOpts) - if err != nil { - return err - } - var role *capi.ACLRole - for _, r := range roles { - if strings.HasSuffix(r.Name, fmt.Sprintf("-%s-acl-role", termGW.Name)) { - role = r - break - } - } - - if role == nil { - log.Info("terminating gateway role not found", "terminatingGatewayName", termGW.Name) - mErr = errors.Join(mErr, fmt.Errorf("terminating gateway role not found for %q", termGW.Name)) - continue - } - - var policyID string - - expectedPolicyName := servicePolicyName(registration.Spec.Service.Name) - role.Policies = slices.DeleteFunc(role.Policies, func(i *capi.ACLRolePolicyLink) bool { - if i.Name == expectedPolicyName { - policyID = i.ID - return true - } - return false - }) - - if policyID == "" { - log.Info("policy not found on terminating gateway role", "policyName", expectedPolicyName, "terminatingGatewayName", termGW.Name) - continue - } - - _, _, err = client.ACL().RoleUpdate(role, writeOpts) - if err != nil { - log.Error(err, "error updating role", "roleName", role.Name) - mErr = errors.Join(mErr, fmt.Errorf("error updating role %q", role.Name)) - continue - } - - _, err = client.ACL().PolicyDelete(policyID, writeOpts) - if err != nil { - log.Error(err, "error deleting service policy", "policyID", policyID, "policyName", expectedPolicyName) - mErr = errors.Join(mErr, fmt.Errorf("error deleting service ACL policy %q", policyID)) - continue - } - } - - return mErr -} - -func servicePolicyName(name string) string { - return fmt.Sprintf("%s-write-policy", name) +func emptyOrDefault(s string) bool { + return s == "" || s == "default" } diff --git a/control-plane/catalog/registration/registrations_controller.go b/control-plane/catalog/registration/registrations_controller.go index 0b6b4bc24e..712aa02522 100644 --- a/control-plane/catalog/registration/registrations_controller.go +++ b/control-plane/catalog/registration/registrations_controller.go @@ -44,12 +44,12 @@ type RegistrationsController struct { Log logr.Logger } -// +kubebuilder:rbac:groups=consul.hashicorp.com,resources=servicerouters,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=consul.hashicorp.com,resources=servicerouters/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=consul.hashicorp.com,resources=registration,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=consul.hashicorp.com,resources=registration/status,verbs=get;update;patch func (r *RegistrationsController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.V(1).WithValues("registration", req.NamespacedName) - log.Info("Reconciling Registaration") + log.Info("Reconciling Registration") registration := &v1alpha1.Registration{} // get the registration @@ -60,18 +60,6 @@ func (r *RegistrationsController) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, client.IgnoreNotFound(err) } - cachedRegistration, ok := r.Cache.get(registration.Spec.Service.Name) - if slices.ContainsFunc(registration.Status.Conditions, func(c v1alpha1.Condition) bool { return c.Type == ConditionDeregistered }) { - if ok && registration.EqualExceptStatus(cachedRegistration) { - log.Info("Registration is in sync") - // registration is already in sync so we do nothing, this happens when consul deregisters a service - // and we update the status to show that consul deregistered it - return ctrl.Result{}, nil - } - } - - log.Info("need to reconcile") - // deletion request if !registration.ObjectMeta.DeletionTimestamp.IsZero() { result := r.handleDeletion(ctx, log, registration) @@ -86,6 +74,19 @@ func (r *RegistrationsController) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, nil } + cachedRegistration, ok := r.Cache.get(registration.Spec.Service.Name) + if slices.ContainsFunc(registration.Status.Conditions, func(c v1alpha1.Condition) bool { return c.Type == ConditionDeregistered }) { + // registration is already in sync so we do nothing, this happens when consul deregisters a service + // and we update the status to show that consul deregistered it + if ok && registration.EqualExceptStatus(cachedRegistration) { + r.Cache.set(registration.Spec.Service.Name, registration) + log.Info("Registration is in sync") + return ctrl.Result{}, nil + } + } + + log.Info("need to reconcile") + // registration request result := r.handleRegistration(ctx, log, registration) err := r.UpdateStatus(ctx, log, registration, result) @@ -113,7 +114,6 @@ func (c *RegistrationsController) watchForDeregistrations(ctx context.Context) { continue } for _, reg := range regList.Items { - err := c.UpdateStatus(context.Background(), c.Log, ®, Result{Registering: false, ConsulDeregistered: true}) if err != nil { c.Log.Error(err, "failed to update Registration status", "name", reg.Name, "namespace", reg.Namespace) @@ -143,48 +143,9 @@ func (r *RegistrationsController) handleRegistration(ctx context.Context, log lo return result } - if r.Cache.aclsEnabled() { - termGWsToUpdate, err := r.terminatingGatewaysToUpdate(ctx, log, registration) - if err != nil { - result.Sync = err - result.ACLUpdate = fmt.Errorf("%w: %s", ErrUpdatingACLRoles, err) - return result - } - - err = r.Cache.updateTermGWACLRole(log, registration, termGWsToUpdate) - if err != nil { - result.Sync = err - result.ACLUpdate = fmt.Errorf("%w: %s", ErrUpdatingACLRoles, err) - return result - } - } return result } -func (r *RegistrationsController) terminatingGatewaysToUpdate(ctx context.Context, log logr.Logger, registration *v1alpha1.Registration) ([]v1alpha1.TerminatingGateway, error) { - termGWList := &v1alpha1.TerminatingGatewayList{} - err := r.Client.List(ctx, termGWList) - if err != nil { - log.Error(err, "error listing terminating gateways") - return nil, err - } - - termGWsToUpdate := make([]v1alpha1.TerminatingGateway, 0, len(termGWList.Items)) - for _, termGW := range termGWList.Items { - if slices.ContainsFunc(termGW.Spec.Services, termGWContainsService(registration)) { - termGWsToUpdate = append(termGWsToUpdate, termGW) - } - } - - return termGWsToUpdate, nil -} - -func termGWContainsService(registration *v1alpha1.Registration) func(v1alpha1.LinkedService) bool { - return func(svc v1alpha1.LinkedService) bool { - return svc.Name == registration.Spec.Service.Name - } -} - func (r *RegistrationsController) handleDeletion(ctx context.Context, log logr.Logger, registration *v1alpha1.Registration) Result { log.Info("Deregistering service") result := Result{Registering: false} @@ -195,22 +156,6 @@ func (r *RegistrationsController) handleDeletion(ctx context.Context, log logr.L return result } - if r.Cache.aclsEnabled() { - termGWsToUpdate, err := r.terminatingGatewaysToUpdate(ctx, log, registration) - if err != nil { - result.Sync = err - result.ACLUpdate = fmt.Errorf("%w: %s", ErrRemovingACLRoles, err) - return result - } - - err = r.Cache.removeTermGWACLRole(log, registration, termGWsToUpdate) - if err != nil { - result.Sync = err - result.ACLUpdate = fmt.Errorf("%w: %s", ErrRemovingACLRoles, err) - return result - } - } - patch := r.RemoveFinalizersPatch(registration, RegistrationFinalizer) err = r.Patch(ctx, registration, patch) if err != nil { @@ -233,10 +178,6 @@ func (r *RegistrationsController) UpdateStatus(ctx context.Context, log logr.Log registration.Status.Conditions = append(registration.Status.Conditions, deregistrationCondition(result)) } - if r.Cache.aclsEnabled() { - registration.Status.Conditions = append(registration.Status.Conditions, aclCondition(result)) - } - err := r.Status().Update(ctx, registration) if err != nil { return err diff --git a/control-plane/catalog/registration/registrations_controller_test.go b/control-plane/catalog/registration/registrations_controller_test.go index ce3dab14f5..afe0e315c5 100644 --- a/control-plane/catalog/registration/registrations_controller_test.go +++ b/control-plane/catalog/registration/registrations_controller_test.go @@ -110,133 +110,6 @@ func TestReconcile_Success(tt *testing.T) { }, }, }, - "registering -- ACLs enabled and policy does not exist": { - registration: &v1alpha1.Registration{ - TypeMeta: metav1.TypeMeta{ - Kind: "Registration", - APIVersion: "consul.hashicorp.com/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-registration", - Finalizers: []string{registration.RegistrationFinalizer}, - }, - Spec: v1alpha1.RegistrationSpec{ - ID: "node-id", - Node: "virtual-node", - Address: "127.0.0.1", - Datacenter: "dc1", - Service: v1alpha1.Service{ - ID: "service-id", - Name: "service-name", - Port: 8080, - Address: "127.0.0.1", - }, - }, - }, - terminatingGateways: []runtime.Object{ - &v1alpha1.TerminatingGateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "terminating-gateway", - }, - Spec: v1alpha1.TerminatingGatewaySpec{ - Services: []v1alpha1.LinkedService{ - { - Name: "service-name", - }, - }, - }, - }, - }, - serverResponseConfig: serverResponseConfig{ - registering: true, - aclEnabled: true, - }, - expectedFinalizers: []string{registration.RegistrationFinalizer}, - expectedConditions: []v1alpha1.Condition{ - { - Type: v1alpha1.ConditionSynced, - Status: v1.ConditionTrue, - Reason: "", - Message: "", - }, - { - Type: registration.ConditionRegistered, - Status: v1.ConditionTrue, - Reason: "", - Message: "", - }, - { - Type: registration.ConditionACLsUpdated, - Status: v1.ConditionTrue, - Reason: "", - Message: "", - }, - }, - }, - "registering -- ACLs enabled and policy does exists": { - registration: &v1alpha1.Registration{ - TypeMeta: metav1.TypeMeta{ - Kind: "Registration", - APIVersion: "consul.hashicorp.com/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-registration", - Finalizers: []string{registration.RegistrationFinalizer}, - }, - Spec: v1alpha1.RegistrationSpec{ - ID: "node-id", - Node: "virtual-node", - Address: "127.0.0.1", - Datacenter: "dc1", - Service: v1alpha1.Service{ - ID: "service-id", - Name: "service-name", - Port: 8080, - Address: "127.0.0.1", - }, - }, - }, - terminatingGateways: []runtime.Object{ - &v1alpha1.TerminatingGateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "terminating-gateway", - }, - Spec: v1alpha1.TerminatingGatewaySpec{ - Services: []v1alpha1.LinkedService{ - { - Name: "service-name", - }, - }, - }, - }, - }, - serverResponseConfig: serverResponseConfig{ - registering: true, - aclEnabled: true, - policyExists: true, - }, - expectedFinalizers: []string{registration.RegistrationFinalizer}, - expectedConditions: []v1alpha1.Condition{ - { - Type: v1alpha1.ConditionSynced, - Status: v1.ConditionTrue, - Reason: "", - Message: "", - }, - { - Type: registration.ConditionRegistered, - Status: v1.ConditionTrue, - Reason: "", - Message: "", - }, - { - Type: registration.ConditionACLsUpdated, - Status: v1.ConditionTrue, - Reason: "", - Message: "", - }, - }, - }, "deregistering": { registration: &v1alpha1.Registration{ TypeMeta: metav1.TypeMeta{ @@ -281,50 +154,6 @@ func TestReconcile_Success(tt *testing.T) { }, expectedConditions: []v1alpha1.Condition{}, }, - "deregistering - ACLs enabled": { - registration: &v1alpha1.Registration{ - TypeMeta: metav1.TypeMeta{ - Kind: "Registration", - APIVersion: "consul.hashicorp.com/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-registration", - Finalizers: []string{registration.RegistrationFinalizer}, - DeletionTimestamp: &deletionTime, - }, - Spec: v1alpha1.RegistrationSpec{ - ID: "node-id", - Node: "virtual-node", - Address: "127.0.0.1", - Datacenter: "dc1", - Service: v1alpha1.Service{ - ID: "service-id", - Name: "service-name", - Port: 8080, - Address: "127.0.0.1", - }, - }, - }, - terminatingGateways: []runtime.Object{ - &v1alpha1.TerminatingGateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "terminating-gateway", - }, - Spec: v1alpha1.TerminatingGatewaySpec{ - Services: []v1alpha1.LinkedService{ - { - Name: "service-name", - }, - }, - }, - }, - }, - serverResponseConfig: serverResponseConfig{ - registering: false, - aclEnabled: true, - }, - expectedConditions: []v1alpha1.Condition{}, - }, } for name, tc := range cases { @@ -436,243 +265,6 @@ func TestReconcile_Failure(tt *testing.T) { }, }, }, - "registering - terminating gateway acl role not found": { - registration: &v1alpha1.Registration{ - TypeMeta: metav1.TypeMeta{ - Kind: "Registration", - APIVersion: "consul.hashicorp.com/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-registration", - Finalizers: []string{registration.RegistrationFinalizer}, - }, - Spec: v1alpha1.RegistrationSpec{ - ID: "node-id", - Node: "virtual-node", - Address: "127.0.0.1", - Datacenter: "dc1", - Service: v1alpha1.Service{ - ID: "service-id", - Name: "service-name", - Port: 8080, - Address: "127.0.0.1", - }, - }, - }, - terminatingGateways: []runtime.Object{ - &v1alpha1.TerminatingGateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "terminating-gateway", - }, - Spec: v1alpha1.TerminatingGatewaySpec{ - Services: []v1alpha1.LinkedService{ - { - Name: "service-name", - }, - }, - }, - }, - }, - serverResponseConfig: serverResponseConfig{ - registering: true, - aclEnabled: true, - temGWRoleMissing: true, - }, - expectedConditions: []v1alpha1.Condition{ - { - Type: v1alpha1.ConditionSynced, - Status: v1.ConditionFalse, - Reason: registration.SyncError, - }, - { - Type: registration.ConditionRegistered, - Status: v1.ConditionTrue, - }, - { - Type: registration.ConditionACLsUpdated, - Status: v1.ConditionFalse, - Reason: registration.ConsulErrorACL, - }, - }, - }, - "registering - error reading policy": { - registration: &v1alpha1.Registration{ - TypeMeta: metav1.TypeMeta{ - Kind: "Registration", - APIVersion: "consul.hashicorp.com/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-registration", - Finalizers: []string{registration.RegistrationFinalizer}, - }, - Spec: v1alpha1.RegistrationSpec{ - ID: "node-id", - Node: "virtual-node", - Address: "127.0.0.1", - Datacenter: "dc1", - Service: v1alpha1.Service{ - ID: "service-id", - Name: "service-name", - Port: 8080, - Address: "127.0.0.1", - }, - }, - }, - terminatingGateways: []runtime.Object{ - &v1alpha1.TerminatingGateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "terminating-gateway", - }, - Spec: v1alpha1.TerminatingGatewaySpec{ - Services: []v1alpha1.LinkedService{ - { - Name: "service-name", - }, - }, - }, - }, - }, - serverResponseConfig: serverResponseConfig{ - registering: true, - aclEnabled: true, - errOnPolicyRead: true, - policyExists: true, - }, - expectedConditions: []v1alpha1.Condition{ - { - Type: v1alpha1.ConditionSynced, - Status: v1.ConditionFalse, - Reason: registration.SyncError, - }, - { - Type: registration.ConditionRegistered, - Status: v1.ConditionTrue, - }, - { - Type: registration.ConditionACLsUpdated, - Status: v1.ConditionFalse, - Reason: registration.ConsulErrorACL, - }, - }, - }, - "registering - policy does not exist - error creating policy": { - registration: &v1alpha1.Registration{ - TypeMeta: metav1.TypeMeta{ - Kind: "Registration", - APIVersion: "consul.hashicorp.com/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-registration", - Finalizers: []string{registration.RegistrationFinalizer}, - }, - Spec: v1alpha1.RegistrationSpec{ - ID: "node-id", - Node: "virtual-node", - Address: "127.0.0.1", - Datacenter: "dc1", - Service: v1alpha1.Service{ - ID: "service-id", - Name: "service-name", - Port: 8080, - Address: "127.0.0.1", - }, - }, - }, - terminatingGateways: []runtime.Object{ - &v1alpha1.TerminatingGateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "terminating-gateway", - }, - Spec: v1alpha1.TerminatingGatewaySpec{ - Services: []v1alpha1.LinkedService{ - { - Name: "service-name", - }, - }, - }, - }, - }, - serverResponseConfig: serverResponseConfig{ - registering: true, - aclEnabled: true, - errOnPolicyWrite: true, - }, - expectedConditions: []v1alpha1.Condition{ - { - Type: v1alpha1.ConditionSynced, - Status: v1.ConditionFalse, - Reason: registration.SyncError, - }, - { - Type: registration.ConditionRegistered, - Status: v1.ConditionTrue, - }, - { - Type: registration.ConditionACLsUpdated, - Status: v1.ConditionFalse, - Reason: registration.ConsulErrorACL, - }, - }, - }, - "registering - error updating role": { - registration: &v1alpha1.Registration{ - TypeMeta: metav1.TypeMeta{ - Kind: "Registration", - APIVersion: "consul.hashicorp.com/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-registration", - Finalizers: []string{registration.RegistrationFinalizer}, - }, - Spec: v1alpha1.RegistrationSpec{ - ID: "node-id", - Node: "virtual-node", - Address: "127.0.0.1", - Datacenter: "dc1", - Service: v1alpha1.Service{ - ID: "service-id", - Name: "service-name", - Port: 8080, - Address: "127.0.0.1", - }, - }, - }, - terminatingGateways: []runtime.Object{ - &v1alpha1.TerminatingGateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "terminating-gateway", - }, - Spec: v1alpha1.TerminatingGatewaySpec{ - Services: []v1alpha1.LinkedService{ - { - Name: "service-name", - }, - }, - }, - }, - }, - serverResponseConfig: serverResponseConfig{ - registering: true, - aclEnabled: true, - errOnRoleUpdate: true, - }, - expectedConditions: []v1alpha1.Condition{ - { - Type: v1alpha1.ConditionSynced, - Status: v1.ConditionFalse, - Reason: registration.SyncError, - }, - { - Type: registration.ConditionRegistered, - Status: v1.ConditionTrue, - }, - { - Type: registration.ConditionACLsUpdated, - Status: v1.ConditionFalse, - Reason: registration.ConsulErrorACL, - }, - }, - }, "deregistering": { registration: &v1alpha1.Registration{ TypeMeta: metav1.TypeMeta{ @@ -727,124 +319,6 @@ func TestReconcile_Failure(tt *testing.T) { }, }, }, - "deregistering - ACLs enabled - terminating-gateway error updating role": { - registration: &v1alpha1.Registration{ - TypeMeta: metav1.TypeMeta{ - Kind: "Registration", - APIVersion: "consul.hashicorp.com/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-registration", - Finalizers: []string{registration.RegistrationFinalizer}, - DeletionTimestamp: &deletionTime, - }, - Spec: v1alpha1.RegistrationSpec{ - ID: "node-id", - Node: "virtual-node", - Address: "127.0.0.1", - Datacenter: "dc1", - Service: v1alpha1.Service{ - ID: "service-id", - Name: "service-name", - Port: 8080, - Address: "127.0.0.1", - }, - }, - }, - terminatingGateways: []runtime.Object{ - &v1alpha1.TerminatingGateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "terminating-gateway", - }, - Spec: v1alpha1.TerminatingGatewaySpec{ - Services: []v1alpha1.LinkedService{ - { - Name: "service-name", - }, - }, - }, - }, - }, - serverResponseConfig: serverResponseConfig{ - aclEnabled: true, - errOnRoleUpdate: true, - }, - expectedConditions: []v1alpha1.Condition{ - { - Type: v1alpha1.ConditionSynced, - Status: v1.ConditionFalse, - Reason: registration.SyncError, - }, - { - Type: registration.ConditionDeregistered, - Status: v1.ConditionTrue, - }, - { - Type: registration.ConditionACLsUpdated, - Status: v1.ConditionFalse, - Reason: registration.ConsulErrorACL, - }, - }, - }, - "deregistering - ACLs enabled - terminating-gateway error deleting policy": { - registration: &v1alpha1.Registration{ - TypeMeta: metav1.TypeMeta{ - Kind: "Registration", - APIVersion: "consul.hashicorp.com/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-registration", - Finalizers: []string{registration.RegistrationFinalizer}, - DeletionTimestamp: &deletionTime, - }, - Spec: v1alpha1.RegistrationSpec{ - ID: "node-id", - Node: "virtual-node", - Address: "127.0.0.1", - Datacenter: "dc1", - Service: v1alpha1.Service{ - ID: "service-id", - Name: "service-name", - Port: 8080, - Address: "127.0.0.1", - }, - }, - }, - terminatingGateways: []runtime.Object{ - &v1alpha1.TerminatingGateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "terminating-gateway", - }, - Spec: v1alpha1.TerminatingGatewaySpec{ - Services: []v1alpha1.LinkedService{ - { - Name: "service-name", - }, - }, - }, - }, - }, - serverResponseConfig: serverResponseConfig{ - aclEnabled: true, - errOnPolicyDelete: true, - }, - expectedConditions: []v1alpha1.Condition{ - { - Type: v1alpha1.ConditionSynced, - Status: v1.ConditionFalse, - Reason: registration.SyncError, - }, - { - Type: registration.ConditionDeregistered, - Status: v1.ConditionTrue, - }, - { - Type: registration.ConditionACLsUpdated, - Status: v1.ConditionFalse, - Reason: registration.ConsulErrorACL, - }, - }, - }, } for name, tc := range cases { diff --git a/control-plane/catalog/registration/result.go b/control-plane/catalog/registration/result.go index 176855c330..0472db5563 100644 --- a/control-plane/catalog/registration/result.go +++ b/control-plane/catalog/registration/result.go @@ -17,7 +17,6 @@ const ( ConditionSynced = "Synced" ConditionRegistered = "Registered" ConditionDeregistered = "Deregistered" - ConditionACLsUpdated = "ACLsUpdated" ) // Status Reasons. @@ -25,7 +24,6 @@ const ( SyncError = "SyncError" ConsulErrorRegistration = "ConsulErrorRegistration" ConsulErrorDeregistration = "ConsulErrorDeregistration" - ConsulErrorACL = "ConsulErrorACL" ConsulDeregistration = "ConsulDeregistration" ) @@ -35,17 +33,16 @@ type Result struct { Sync error Registration error Deregistration error - ACLUpdate error Finalizer error } func (r Result) hasErrors() bool { - return r.Sync != nil || r.Registration != nil || r.ACLUpdate != nil || r.Finalizer != nil + return r.Sync != nil || r.Registration != nil || r.Finalizer != nil } func (r Result) errors() error { var err error - err = errors.Join(err, r.Sync, r.Registration, r.ACLUpdate, r.Finalizer) + err = errors.Join(err, r.Sync, r.Registration, r.Finalizer) return err } @@ -110,31 +107,3 @@ func deregistrationCondition(result Result) v1alpha1.Condition { LastTransitionTime: metav1.Now(), } } - -func aclCondition(result Result) v1alpha1.Condition { - if result.ACLUpdate != nil { - return v1alpha1.Condition{ - Type: ConditionACLsUpdated, - Status: corev1.ConditionFalse, - Reason: ConsulErrorACL, - Message: result.ACLUpdate.Error(), - LastTransitionTime: metav1.Now(), - } - } - - if result.ConsulDeregistered { - return v1alpha1.Condition{ - Type: ConditionACLsUpdated, - Status: corev1.ConditionFalse, - Reason: ConsulDeregistration, - Message: "Consul deregistered this service, acls were not removed", - LastTransitionTime: metav1.Now(), - } - } - - return v1alpha1.Condition{ - Type: ConditionACLsUpdated, - Status: corev1.ConditionTrue, - LastTransitionTime: metav1.Now(), - } -} diff --git a/control-plane/config/rbac/role.yaml b/control-plane/config/rbac/role.yaml index c2ad591c4f..3eb003fae9 100644 --- a/control-plane/config/rbac/role.yaml +++ b/control-plane/config/rbac/role.yaml @@ -205,6 +205,26 @@ rules: - get - patch - update +- apiGroups: + - consul.hashicorp.com + resources: + - registration + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - consul.hashicorp.com + resources: + - registration/status + verbs: + - get + - patch + - update - apiGroups: - consul.hashicorp.com resources: diff --git a/control-plane/controllers/configentries/configentry_controller_test.go b/control-plane/controllers/configentries/configentry_controller_test.go index 0b57e6e50b..db38e950cc 100644 --- a/control-plane/controllers/configentries/configentry_controller_test.go +++ b/control-plane/controllers/configentries/configentry_controller_test.go @@ -15,6 +15,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" capi "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -543,48 +544,59 @@ func TestConfigEntryControllers_createsConfigEntry(t *testing.T) { } for _, c := range cases { - t.Run(c.kubeKind, func(t *testing.T) { - req := require.New(t) - ctx := context.Background() + for _, secure := range []bool{true, false} { + t.Run(fmt.Sprintf("%s: %t", c.kubeKind, secure), func(t *testing.T) { + req := require.New(t) + ctx := context.Background() + + s := runtime.NewScheme() + s.AddKnownTypes(v1alpha1.GroupVersion, c.configEntryResource) + fakeClient := fake.NewClientBuilder().WithScheme(s).WithObjects(c.configEntryResource).WithStatusSubresource(c.configEntryResource).Build() + + var cb testutil.ServerConfigCallback + if secure { + adminToken := "123e4567-e89b-12d3-a456-426614174000" + cb = func(c *testutil.TestServerConfig) { + c.ACL.Enabled = true + c.ACL.Tokens.InitialManagement = adminToken + } + } - s := runtime.NewScheme() - s.AddKnownTypes(v1alpha1.GroupVersion, c.configEntryResource) - fakeClient := fake.NewClientBuilder().WithScheme(s).WithObjects(c.configEntryResource).WithStatusSubresource(c.configEntryResource).Build() + testClient := test.TestServerWithMockConnMgrWatcher(t, cb) + testClient.TestServer.WaitForServiceIntentions(t) + consulClient := testClient.APIClient - testClient := test.TestServerWithMockConnMgrWatcher(t, nil) - testClient.TestServer.WaitForServiceIntentions(t) - consulClient := testClient.APIClient + for _, configEntry := range c.consulPrereqs { + written, _, err := consulClient.ConfigEntries().Set(configEntry, nil) + req.NoError(err) + req.True(written) + } - for _, configEntry := range c.consulPrereqs { - written, _, err := consulClient.ConfigEntries().Set(configEntry, nil) + r := c.reconciler(fakeClient, testClient.Cfg, testClient.Watcher, logrtest.New(t)) + namespacedName := types.NamespacedName{ + Namespace: kubeNS, + Name: c.configEntryResource.KubernetesName(), + } + resp, err := r.Reconcile(ctx, ctrl.Request{ + NamespacedName: namespacedName, + }) req.NoError(err) - req.True(written) - } - - r := c.reconciler(fakeClient, testClient.Cfg, testClient.Watcher, logrtest.New(t)) - namespacedName := types.NamespacedName{ - Namespace: kubeNS, - Name: c.configEntryResource.KubernetesName(), - } - resp, err := r.Reconcile(ctx, ctrl.Request{ - NamespacedName: namespacedName, - }) - req.NoError(err) - req.False(resp.Requeue) + req.False(resp.Requeue) - cfg, _, err := consulClient.ConfigEntries().Get(c.consulKind, c.configEntryResource.ConsulName(), nil) - req.NoError(err) - req.Equal(c.configEntryResource.ConsulName(), cfg.GetName()) - c.compare(t, cfg) + cfg, _, err := consulClient.ConfigEntries().Get(c.consulKind, c.configEntryResource.ConsulName(), nil) + req.NoError(err) + req.Equal(c.configEntryResource.ConsulName(), cfg.GetName()) + c.compare(t, cfg) - // Check that the status is "synced". - err = fakeClient.Get(ctx, namespacedName, c.configEntryResource) - req.NoError(err) - req.Equal(corev1.ConditionTrue, c.configEntryResource.SyncedConditionStatus()) + // Check that the status is "synced". + err = fakeClient.Get(ctx, namespacedName, c.configEntryResource) + req.NoError(err) + req.Equal(corev1.ConditionTrue, c.configEntryResource.SyncedConditionStatus()) - // Check that the finalizer is added. - req.Contains(c.configEntryResource.Finalizers(), FinalizerName) - }) + // Check that the finalizer is added. + req.Contains(c.configEntryResource.Finalizers(), FinalizerName) + }) + } } } diff --git a/control-plane/controllers/configentries/terminatinggateway_controller.go b/control-plane/controllers/configentries/terminatinggateway_controller.go index f8e4a0bc0b..ec329bd17c 100644 --- a/control-plane/controllers/configentries/terminatinggateway_controller.go +++ b/control-plane/controllers/configentries/terminatinggateway_controller.go @@ -4,33 +4,122 @@ package configentries import ( + "bytes" "context" + "errors" + "fmt" + "strings" + "text/template" + mapset "github.com/deckarep/golang-set/v2" "github.com/go-logr/logr" + capi "github.com/hashicorp/consul/api" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" consulv1alpha1 "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" + "github.com/hashicorp/consul-k8s/control-plane/consul" ) var _ Controller = (*TerminatingGatewayController)(nil) +const terminatingGatewayByLinkedServiceName = "linkedServiceName" + // TerminatingGatewayController is the controller for TerminatingGateway resources. type TerminatingGatewayController struct { client.Client FinalizerPatcher + + NamespacesEnabled bool + Log logr.Logger Scheme *runtime.Scheme ConfigEntryController *ConfigEntryController } +func init() { + servicePolicyTpl = template.Must(template.New("root").Parse(strings.TrimSpace(servicePolicyRulesTpl))) + wildcardPolicyTpl = template.Must(template.New("root").Parse(strings.TrimSpace(wildcardPolicyRulesTpl))) +} + +type templateArgs struct { + Namespace string + ServiceName string + EnableNamespaces bool +} + +var ( + servicePolicyTpl *template.Template + servicePolicyRulesTpl = ` +{{- if .EnableNamespaces }} +namespace "{{.Namespace}}" { +{{- end }} + service "{{.ServiceName}}" { + policy = "write" + } +{{- if .EnableNamespaces }} +} +{{- end }} +` + + wildcardPolicyTpl *template.Template + wildcardPolicyRulesTpl = ` +{{- if .EnableNamespaces }} +namespace "{{.Namespace}}" { +{{- end }} + service_prefix "" { + policy = "write" + } +{{- if .EnableNamespaces }} +} +{{- end }} +` +) + // +kubebuilder:rbac:groups=consul.hashicorp.com,resources=terminatinggateways,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=consul.hashicorp.com,resources=terminatinggateways/status,verbs=get;update;patch func (r *TerminatingGatewayController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - return r.ConfigEntryController.ReconcileEntry(ctx, r, req, &consulv1alpha1.TerminatingGateway{}) + log := r.Log.V(1).WithValues("terminating-gateway", req.NamespacedName) + log.Info("Reconciling TerminatingGateway") + termGW := &consulv1alpha1.TerminatingGateway{} + // get the registration + if err := r.Client.Get(ctx, req.NamespacedName, termGW); err != nil { + if !k8serrors.IsNotFound(err) { + log.Error(err, "unable to get terminating-gateway") + } + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // creation/modification + enabled, err := r.aclsEnabled() + if err != nil { + log.Error(err, "error checking if acls are enabled") + return ctrl.Result{}, err + } + + if enabled { + err := r.updateACls(log, termGW) + if err != nil { + log.Error(err, "error updating terminating-gateway roles") + r.UpdateStatusFailedToSetACLs(ctx, termGW, err) + return ctrl.Result{}, err + } + + termGW.SetACLStatusConditon(corev1.ConditionTrue, "", "") + err = r.UpdateStatus(ctx, termGW) + if err != nil { + log.Error(err, "error updating terminating-gateway status") + return ctrl.Result{}, err + } + } + + return r.ConfigEntryController.ReconcileEntry(ctx, r, req, termGW) } func (r *TerminatingGatewayController) Logger(name types.NamespacedName) logr.Logger { @@ -41,6 +130,234 @@ func (r *TerminatingGatewayController) UpdateStatus(ctx context.Context, obj cli return r.Status().Update(ctx, obj, opts...) } -func (r *TerminatingGatewayController) SetupWithManager(mgr ctrl.Manager) error { +func (r *TerminatingGatewayController) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + // setup the index to lookup registrations by service name + if err := mgr.GetFieldIndexer().IndexField(ctx, &v1alpha1.TerminatingGateway{}, terminatingGatewayByLinkedServiceName, termGWLinkedServiceIndexer); err != nil { + return err + } + return setupWithManager(mgr, &consulv1alpha1.TerminatingGateway{}, r) } + +func termGWLinkedServiceIndexer(o client.Object) []string { + termGW := o.(*v1alpha1.TerminatingGateway) + names := make([]string, 0, len(termGW.Spec.Services)) + for _, service := range termGW.Spec.Services { + names = append(names, service.Name) + } + + return names +} + +func (r *TerminatingGatewayController) UpdateStatusFailedToSetACLs(ctx context.Context, termGW *consulv1alpha1.TerminatingGateway, err error) { + termGW.SetSyncedCondition(corev1.ConditionFalse, consulv1alpha1.TerminatingGatewayFailedToSetACLs, err.Error()) + termGW.SetACLStatusConditon(corev1.ConditionFalse, consulv1alpha1.TerminatingGatewayFailedToSetACLs, err.Error()) + if err := r.UpdateStatus(ctx, termGW); err != nil { + r.Log.Error(err, "error updating status") + } +} + +func (r *TerminatingGatewayController) aclsEnabled() (bool, error) { + state, err := r.ConfigEntryController.ConsulServerConnMgr.State() + if err != nil { + return false, err + } + return state.Token != "", nil +} + +func (r *TerminatingGatewayController) updateACls(log logr.Logger, termGW *consulv1alpha1.TerminatingGateway) error { + client, err := consul.NewClientFromConnMgr(r.ConfigEntryController.ConsulClientConfig, r.ConfigEntryController.ConsulServerConnMgr) + if err != nil { + return err + } + + roles, _, err := client.ACL().RoleList(nil) + if err != nil { + return err + } + + terminatingGatewayRoleID := "" + for _, role := range roles { + // terminating gateway roles are always of the form ${INSTALL_NAME}-consul-${GATEWAY_NAME}-acl-role + if strings.HasSuffix(role.Name, fmt.Sprintf("%s-acl-role", termGW.Name)) { + terminatingGatewayRoleID = role.ID + break + } + } + + if terminatingGatewayRoleID == "" { + return errors.New("terminating gateway role not found") + } + + terminatingGatewayRole, _, err := client.ACL().RoleRead(terminatingGatewayRoleID, nil) + if err != nil { + return err + } + + var terminatingGatewayPolicy *capi.ACLRolePolicyLink + + for _, policy := range terminatingGatewayRole.Policies { + // terminating gateway policies are always of the form ${GATEWAY_NAME}-policy + if policy.Name == fmt.Sprintf("%s-policy", termGW.Name) { + terminatingGatewayPolicy = policy + break + } + } + + var termGWPoliciesToKeep []*capi.ACLRolePolicyLink + var termGWPoliciesToRemove []*capi.ACLRolePolicyLink + + existingTermGWPolicies := mapset.NewSet[string]() + + for _, policy := range terminatingGatewayRole.Policies { + existingTermGWPolicies.Add(policy.Name) + } + + if termGW.ObjectMeta.DeletionTimestamp.IsZero() { + termGWPoliciesToKeep, termGWPoliciesToRemove, err = r.handleModificationForPolicies(log, client, existingTermGWPolicies, termGW.Spec.Services) + if err != nil { + return err + } + } else { + termGWPoliciesToKeep, termGWPoliciesToRemove = handleDeletionForPolicies(termGW.Spec.Services) + } + + termGWPoliciesToKeep = append(termGWPoliciesToKeep, terminatingGatewayPolicy) + terminatingGatewayRole.Policies = termGWPoliciesToKeep + + _, _, err = client.ACL().RoleUpdate(terminatingGatewayRole, nil) + if err != nil { + return err + } + + err = r.conditionallyDeletePolicies(log, client, termGWPoliciesToRemove, termGW.Name) + if err != nil { + return err + } + + return nil +} + +func handleDeletionForPolicies(services []v1alpha1.LinkedService) ([]*capi.ACLRolePolicyLink, []*capi.ACLRolePolicyLink) { + var termGWPoliciesToRemove []*capi.ACLRolePolicyLink + for _, service := range services { + termGWPoliciesToRemove = append(termGWPoliciesToRemove, &capi.ACLRolePolicyLink{Name: servicePolicyName(service.Name, defaultIfEmpty(service.Namespace))}) + } + return nil, termGWPoliciesToRemove +} + +func (r *TerminatingGatewayController) handleModificationForPolicies(log logr.Logger, client *capi.Client, existingTermGWPolicies mapset.Set[string], services []v1alpha1.LinkedService) ([]*capi.ACLRolePolicyLink, []*capi.ACLRolePolicyLink, error) { + // add one to length to include the terminating-gateway policy for itself + termGWPoliciesToKeep := make([]*capi.ACLRolePolicyLink, 0, len(services)+1) + termGWPoliciesToRemove := make([]*capi.ACLRolePolicyLink, 0, len(services)) + + termGWPoliciesToKeepNames := mapset.NewSet[string]() + for _, service := range services { + existingPolicy, _, err := client.ACL().PolicyReadByName(servicePolicyName(service.Name, defaultIfEmpty(service.Namespace)), &capi.QueryOptions{}) + if err != nil { + log.Error(err, "error reading policy") + return nil, nil, err + } + + if existingPolicy == nil { + policyTemplate := getPolicyTemplateFor(service.Name) + var data bytes.Buffer + if err := policyTemplate.Execute(&data, templateArgs{ + EnableNamespaces: r.NamespacesEnabled, + Namespace: defaultIfEmpty(service.Namespace), + ServiceName: service.Name, + }); err != nil { + // just panic if we can't compile the simple template + // as it means something else is going severly wrong. + panic(err) + } + + _, _, err = client.ACL().PolicyCreate(&capi.ACLPolicy{ + Name: servicePolicyName(service.Name, defaultIfEmpty(service.Namespace)), + Rules: data.String(), + }, nil) + if err != nil { + return nil, nil, err + } + } + + termGWPoliciesToKeep = append(termGWPoliciesToKeep, &capi.ACLRolePolicyLink{Name: servicePolicyName(service.Name, defaultIfEmpty(service.Namespace))}) + termGWPoliciesToKeepNames.Add(servicePolicyName(service.Name, defaultIfEmpty(service.Namespace))) + } + + for _, policy := range existingTermGWPolicies.Difference(termGWPoliciesToKeepNames).ToSlice() { + termGWPoliciesToRemove = append(termGWPoliciesToRemove, &capi.ACLRolePolicyLink{Name: policy}) + } + + return termGWPoliciesToKeep, termGWPoliciesToRemove, nil +} + +func (r *TerminatingGatewayController) conditionallyDeletePolicies(log logr.Logger, consulClient *capi.Client, policies []*capi.ACLRolePolicyLink, termGWName string) error { + policiesToDelete := make([]*capi.ACLRolePolicyLink, 0, len(policies)) + var mErr error + for _, policy := range policies { + termGWList := &v1alpha1.TerminatingGatewayList{} + serviceName := serviceNameFromPolicy(policy.Name) + + if err := r.Client.List(context.Background(), termGWList, client.MatchingFields{terminatingGatewayByLinkedServiceName: serviceName}); err != nil { + log.Error(err, "failed to lookup terminating gateway list for service", serviceName) + mErr = errors.Join(mErr, fmt.Errorf("failed to lookup terminating gateway list for service %q: %w", serviceName, err)) + continue + } + if len(termGWList.Items) == 0 { + policiesToDelete = append(policiesToDelete, policy) + } + } + + for _, policy := range policiesToDelete { + // don't delete the policy for the gateway itself + if policy.Name == fmt.Sprintf("%s-policy", termGWName) { + continue + } + + policy, _, err := consulClient.ACL().PolicyReadByName(policy.Name, nil) + if err != nil { + log.Error(err, "failed to lookup policy by name from consul", policy.Name) + mErr = errors.Join(mErr, fmt.Errorf("error reading policy %q: %w", policy.Name, err)) + continue + } + + _, err = consulClient.ACL().PolicyDelete(policy.ID, nil) + if err != nil { + log.Error(err, "failed to delete policy from consul", policy.Name) + mErr = errors.Join(mErr, fmt.Errorf("error delete policy %q: %w", policy.Name, err)) + } + } + + return mErr +} + +func getPolicyTemplateFor(service string) *template.Template { + if service == "*" { + return wildcardPolicyTpl + } + return servicePolicyTpl +} + +func defaultIfEmpty(s string) string { + if s == "" { + return "default" + } + return s +} + +func servicePolicyName(name, namespace string) string { + if name == "*" { + return fmt.Sprintf("%s-wildcard-write-policy", namespace) + } + + return fmt.Sprintf("%s-%s-write-policy", namespace, name) +} + +func serviceNameFromPolicy(policyName string) string { + // remove the namespace from the beginning of the string + _, n, _ := strings.Cut(policyName, "-") + + // remove the write policy suffix + return strings.TrimSuffix(n, "-write-policy") +} diff --git a/control-plane/subcommand/inject-connect/v1controllers.go b/control-plane/subcommand/inject-connect/v1controllers.go index 6e480978c0..eb87bdd047 100644 --- a/control-plane/subcommand/inject-connect/v1controllers.go +++ b/control-plane/subcommand/inject-connect/v1controllers.go @@ -254,8 +254,9 @@ func (c *Command) configureV1Controllers(ctx context.Context, mgr manager.Manage ConfigEntryController: configEntryReconciler, Client: mgr.GetClient(), Log: ctrl.Log.WithName("controller").WithName(apicommon.TerminatingGateway), + NamespacesEnabled: c.flagEnableNamespaces, Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { + }).SetupWithManager(ctx, mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", apicommon.TerminatingGateway) return err }