From cf16128edd5576020d7a2f2b61c37c8a9bc70ab0 Mon Sep 17 00:00:00 2001 From: sarahalsmiller <100602640+sarahalsmiller@users.noreply.github.com> Date: Tue, 26 Nov 2024 11:52:03 -0600 Subject: [PATCH] [NET-11187] Nightly openshift basic test (#4424) * basic test cases * test fixtures * working openshift test * openshift test packages * update Makefile * Correct name of added Make target * Correct import blocking * Make assertions more robust, clean up code/comments * Add a PR trigger for acceptance tests * Use different source for branch name provided to workflow * Rename PR-triggered workflow * Use local checkout of Helm chart for installation * Use workspace from environment * Add the hashicorp helm repo as part of test setup We will need to consume the cloned repo chart in the future, but I'm trying to get something working first * new chart path * Remove unused code, retry gateway connection, improve logging * Remove PR trigger for nightly acceptance tests * move cleanup calls before the error so they'll clean up even if it errrors * Update acceptance/tests/openshift/main_test.go * update flags to use config * readd pr trigger test to validate change * add license mounting config * fix cert reference --------- Co-authored-by: Nathan Coleman --- Makefile | 5 + .../openshift_acceptance_test_packages.yaml | 5 + acceptance/framework/config/config.go | 1 + acceptance/framework/flags/flags.go | 8 +- .../cases/openshift/basic/backend.yaml | 59 +++++++ .../cases/openshift/basic/frontend.yaml | 62 +++++++ .../cases/openshift/basic/gateway.yaml | 15 ++ .../cases/openshift/basic/intentions.yaml | 23 +++ .../fixtures/cases/openshift/basic/route.yaml | 30 ++++ .../tests/openshift/basic_openshift_test.go | 163 ++++++++++++++++++ acceptance/tests/openshift/main_test.go | 23 +++ control-plane/go.mod | 2 +- 12 files changed, 394 insertions(+), 2 deletions(-) create mode 100644 acceptance/ci-inputs/openshift_acceptance_test_packages.yaml create mode 100644 acceptance/tests/fixtures/cases/openshift/basic/backend.yaml create mode 100644 acceptance/tests/fixtures/cases/openshift/basic/frontend.yaml create mode 100644 acceptance/tests/fixtures/cases/openshift/basic/gateway.yaml create mode 100644 acceptance/tests/fixtures/cases/openshift/basic/intentions.yaml create mode 100644 acceptance/tests/fixtures/cases/openshift/basic/route.yaml create mode 100644 acceptance/tests/openshift/basic_openshift_test.go create mode 100644 acceptance/tests/openshift/main_test.go diff --git a/Makefile b/Makefile index 15f4706943..725599e443 100644 --- a/Makefile +++ b/Makefile @@ -334,6 +334,11 @@ eks-test-packages: ## eks test packages aks-test-packages: ## aks test packages @./control-plane/build-support/scripts/set_test_package_matrix.sh "acceptance/ci-inputs/aks_acceptance_test_packages.yaml" + +.PHONY: openshift-test-packages +openshift-test-packages: ## openshift test packages + @./control-plane/build-support/scripts/set_test_package_matrix.sh "acceptance/ci-inputs/openshift_acceptance_test_packages.yaml" + .PHONY: go-mod-tidy go-mod-tidy: ## Recursively run go mod tidy on all subdirectories @./control-plane/build-support/scripts/mod_tidy.sh diff --git a/acceptance/ci-inputs/openshift_acceptance_test_packages.yaml b/acceptance/ci-inputs/openshift_acceptance_test_packages.yaml new file mode 100644 index 0000000000..5a469d1032 --- /dev/null +++ b/acceptance/ci-inputs/openshift_acceptance_test_packages.yaml @@ -0,0 +1,5 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +# Cloud package is not included in test suite as it is triggered from a non consul-k8s repo and requires HCP credentials +- {runner: 0, test-packages: "openshift"} \ No newline at end of file diff --git a/acceptance/framework/config/config.go b/acceptance/framework/config/config.go index 370e276bc7..ef0a111109 100644 --- a/acceptance/framework/config/config.go +++ b/acceptance/framework/config/config.go @@ -107,6 +107,7 @@ type TestConfig struct { UseGKE bool UseGKEAutopilot bool UseKind bool + UseOpenshift bool helmChartPath string } diff --git a/acceptance/framework/flags/flags.go b/acceptance/framework/flags/flags.go index c956c3f7e3..421afd35cf 100644 --- a/acceptance/framework/flags/flags.go +++ b/acceptance/framework/flags/flags.go @@ -10,8 +10,9 @@ import ( "strings" "sync" - "github.com/hashicorp/consul-k8s/acceptance/framework/config" "github.com/hashicorp/go-version" + + "github.com/hashicorp/consul-k8s/acceptance/framework/config" ) type TestFlags struct { @@ -57,6 +58,7 @@ type TestFlags struct { flagUseGKE bool flagUseGKEAutopilot bool flagUseKind bool + flagUseOpenshift bool flagDisablePeering bool @@ -154,6 +156,9 @@ func (t *TestFlags) init() { flag.BoolVar(&t.flagUseKind, "use-kind", false, "If true, the tests will assume they are running against a local kind cluster(s).") + flag.BoolVar(&t.flagUseOpenshift, "use-openshift", false, + "If true, the tests will assume they are running against an openshift cluster(s).") + flag.BoolVar(&t.flagDisablePeering, "disable-peering", false, "If true, the peering tests will not run.") @@ -246,6 +251,7 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig { UseGKE: t.flagUseGKE, UseGKEAutopilot: t.flagUseGKEAutopilot, UseKind: t.flagUseKind, + UseOpenshift: t.flagUseOpenshift, } return c diff --git a/acceptance/tests/fixtures/cases/openshift/basic/backend.yaml b/acceptance/tests/fixtures/cases/openshift/basic/backend.yaml new file mode 100644 index 0000000000..97a799082e --- /dev/null +++ b/acceptance/tests/fixtures/cases/openshift/basic/backend.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: backend +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: backend + namespace: backend +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend + namespace: backend +spec: + selector: + matchLabels: + app: backend + replicas: 1 + template: + metadata: + labels: + app: backend + annotations: + consul.hashicorp.com/connect-inject: "true" + spec: + serviceAccountName: backend + containers: + - name: backend + image: nicholasjackson/fake-service:v0.26.0 + ports: + - containerPort: 8080 + env: + - name: LISTEN_ADDR + value: "0.0.0.0:8080" + - name: NAME + value: backend +--- +apiVersion: v1 +kind: Service +metadata: + name: backend + namespace: backend +spec: + type: ClusterIP + selector: + app: backend + ports: + - port: 8080 +--- +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceDefaults +metadata: + name: backend + namespace: backend +spec: + protocol: http diff --git a/acceptance/tests/fixtures/cases/openshift/basic/frontend.yaml b/acceptance/tests/fixtures/cases/openshift/basic/frontend.yaml new file mode 100644 index 0000000000..e5dfbcb6bd --- /dev/null +++ b/acceptance/tests/fixtures/cases/openshift/basic/frontend.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: frontend +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: frontend + namespace: frontend +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: frontend + namespace: frontend +spec: + selector: + matchLabels: + app: frontend + replicas: 1 + template: + metadata: + labels: + app: frontend + annotations: + consul.hashicorp.com/connect-inject: "true" + spec: + serviceAccountName: frontend + containers: + - name: frontend + image: nicholasjackson/fake-service:v0.26.0 + ports: + - containerPort: 8080 + env: + - name: LISTEN_ADDR + value: "0.0.0.0:8080" + - name: NAME + value: frontend + - name: UPSTREAM_URIS + value: 'http://backend.backend:8080' + +--- +apiVersion: v1 +kind: Service +metadata: + name: frontend + namespace: frontend +spec: + type: ClusterIP + selector: + app: frontend + ports: + - port: 8080 +--- +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceDefaults +metadata: + name: frontend + namespace: frontend +spec: + protocol: http diff --git a/acceptance/tests/fixtures/cases/openshift/basic/gateway.yaml b/acceptance/tests/fixtures/cases/openshift/basic/gateway.yaml new file mode 100644 index 0000000000..39bedf9d3f --- /dev/null +++ b/acceptance/tests/fixtures/cases/openshift/basic/gateway.yaml @@ -0,0 +1,15 @@ +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: api-gateway + namespace: consul +spec: + gatewayClassName: consul + listeners: + - name: https + protocol: HTTPS + port: 443 + tls: + certificateRefs: + - name: consul-server-cert + namespace: consul diff --git a/acceptance/tests/fixtures/cases/openshift/basic/intentions.yaml b/acceptance/tests/fixtures/cases/openshift/basic/intentions.yaml new file mode 100644 index 0000000000..45f19aa58c --- /dev/null +++ b/acceptance/tests/fixtures/cases/openshift/basic/intentions.yaml @@ -0,0 +1,23 @@ +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceIntentions +metadata: + name: to-backend-default + namespace: default +spec: + destination: + name: backend + sources: + - name: frontend + action: allow +--- +apiVersion: consul.hashicorp.com/v1alpha1 +kind: ServiceIntentions +metadata: + name: to-frontend-default + namespace: default +spec: + destination: + name: frontend + sources: + - name: api-gateway + action: allow diff --git a/acceptance/tests/fixtures/cases/openshift/basic/route.yaml b/acceptance/tests/fixtures/cases/openshift/basic/route.yaml new file mode 100644 index 0000000000..2cd0b9815c --- /dev/null +++ b/acceptance/tests/fixtures/cases/openshift/basic/route.yaml @@ -0,0 +1,30 @@ +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: frontend-route-default + namespace: consul +spec: + parentRefs: + - name: api-gateway + rules: + - backendRefs: + - kind: Service + name: frontend + namespace: frontend + port: 8080 + +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: ReferenceGrant +metadata: + name: service-grant + namespace: frontend +spec: + from: + - group: gateway.networking.k8s.io + kind: HTTPRoute + namespace: consul + to: + - group: "" + kind: Service + name: frontend diff --git a/acceptance/tests/openshift/basic_openshift_test.go b/acceptance/tests/openshift/basic_openshift_test.go new file mode 100644 index 0000000000..ab39d9b3e6 --- /dev/null +++ b/acceptance/tests/openshift/basic_openshift_test.go @@ -0,0 +1,163 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package openshift + +import ( + "context" + "crypto/tls" + "encoding/json" + "net/http" + "os/exec" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/hashicorp/consul/sdk/testutil/retry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/types" + logf "sigs.k8s.io/controller-runtime/pkg/log" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" + "github.com/hashicorp/consul-k8s/acceptance/framework/logger" +) + +// Test that api gateway basic functionality works in a default installation and a secure installation. +func TestOpenshift_Basic(t *testing.T) { + cfg := suite.Config() + + cmd := exec.Command("helm", "repo", "add", "hashicorp", "https://helm.releases.hashicorp.com") + output, err := cmd.CombinedOutput() + require.NoErrorf(t, err, "failed to add hashicorp helm repo: %s", string(output)) + + // FUTURE for some reason NewHelmCluster creates a consul server pod that runs as root which + // isn't allowed in OpenShift. In order to test OpenShift properly, we have to call helm and k8s + // directly to bypass. Ideally we would just fix the framework that is running the pod as root. + cmd = exec.Command("kubectl", "create", "namespace", "consul") + output, err = cmd.CombinedOutput() + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + cmd = exec.Command("kubectl", "delete", "namespace", "consul") + output, err = cmd.CombinedOutput() + assert.NoErrorf(t, err, "failed to delete namespace: %s", string(output)) + }) + + require.NoErrorf(t, err, "failed to add hashicorp helm repo: %s", string(output)) + + cmd = exec.Command("kubectl", "create", "secret", "generic", + "consul-ent-license", + "--namespace", "consul", + `--from-literal=key=`+cfg.EnterpriseLicense) + output, err = cmd.CombinedOutput() + require.NoErrorf(t, err, "failed to add consul enterprise license: %s", string(output)) + + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + cmd = exec.Command("kubectl", "delete", "secret", "consul-ent-license") + output, err = cmd.CombinedOutput() + assert.NoErrorf(t, err, "failed to delete secret: %s", string(output)) + }) + + chartPath := "../../../charts/consul" + cmd = exec.Command("helm", "upgrade", "--install", "consul", chartPath, + "--namespace", "consul", + "--set", "global.name=consul", + "--set", "connectInject.enabled=true", + "--set", "connectInject.transparentProxy.defaultEnabled=false", + "--set", "connectInject.apiGateway.managedGatewayClass.mapPrivilegedContainerPorts=8000", + "--set", "global.acls.manageSystemACLs=true", + "--set", "global.tls.enabled=true", + "--set", "global.tls.enableAutoEncrypt=true", + "--set", "global.openshift.enabled=true", + "--set", "global.image="+cfg.ConsulImage, + "--set", "global.imageK8S="+cfg.ConsulK8SImage, + "--set", "global.imageConsulDataplane="+cfg.ConsulDataplaneImage, + "--set", "global.enterpriseLicense.secretName=consul-ent-license", + "--set", "global.enterpriseLicense.secretKey=key", + ) + output, err = cmd.CombinedOutput() + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + cmd := exec.Command("helm", "uninstall", "consul", "--namespace", "consul") + output, err := cmd.CombinedOutput() + require.NoErrorf(t, err, "failed to uninstall consul: %s", string(output)) + }) + + require.NoErrorf(t, err, "failed to install consul: %s", string(output)) + + // this is normally called by the environment, but because we have to bypass we have to call it explicitly + logf.SetLogger(logr.New(nil)) + logger.Log(t, "creating resources for OpenShift test") + + cmd = exec.Command("kubectl", "apply", "-f", "../fixtures/cases/openshift/basic") + output, err = cmd.CombinedOutput() + helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() { + cmd := exec.Command("kubectl", "delete", "-f", "../fixtures/cases/openshift/basic") + output, err := cmd.CombinedOutput() + assert.NoErrorf(t, err, "failed to delete resources: %s", string(output)) + }) + + require.NoErrorf(t, err, "failed to create resources: %s", string(output)) + + // Grab a kubernetes client so that we can verify binding + // behavior prior to issuing requests through the gateway. + ctx := suite.Environment().DefaultContext(t) + k8sClient := ctx.ControllerRuntimeClient(t) + + // Get the public IP address of the API gateway that we created from its status. + // + // On startup, the controller can take upwards of 1m to perform leader election, + // so we may need to wait a long time for the reconcile loop to run (hence the timeout). + var gatewayIP string + counter := &retry.Counter{Count: 120, Wait: 2 * time.Second} + retry.RunWith(counter, t, func(r *retry.R) { + var gateway gwv1beta1.Gateway + err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "api-gateway", Namespace: "consul"}, &gateway) + require.NoError(r, err) + + require.Len(r, gateway.Status.Addresses, 1) + gatewayIP = gateway.Status.Addresses[0].Value + }) + logger.Log(t, "API gateway is reachable at:", gatewayIP) + + // Verify that we can reach the services that we created in the mesh + // via the API gateway that we created. + // + // The request goes Gateway --> Frontend --> Backend + client := &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }} + + var resp *http.Response + counter = &retry.Counter{Count: 120, Wait: 2 * time.Second} + retry.RunWith(counter, t, func(r *retry.R) { + resp, err = client.Get("https://" + gatewayIP) + require.NoErrorf(r, err, "request to API gateway failed: %s", err) + assert.Equalf(r, resp.StatusCode, http.StatusOK, "request to API gateway returned failure code: %d", resp.StatusCode) + }) + + var body struct { + Body string `json:"body"` + Code int `json:"code"` + Name string `json:"name"` + UpstreamCalls map[string]struct { + Body string `json:"body"` + Code int `json:"code"` + Name string `json:"name"` + } `json:"upstream_calls"` + URI string `json:"uri"` + } + + require.NoError(t, json.NewDecoder(resp.Body).Decode(&body)) + assert.Equal(t, "Hello World", body.Body) + assert.Equal(t, 200, body.Code) + assert.Equal(t, "frontend", body.Name) + assert.Equal(t, "/", body.URI) + + require.Len(t, body.UpstreamCalls, 1) + require.Contains(t, body.UpstreamCalls, "http://backend.backend:8080") + + backend := body.UpstreamCalls["http://backend.backend:8080"] + assert.Equal(t, "Hello World", body.Body) + assert.Equal(t, 200, backend.Code) + assert.Equal(t, "backend", backend.Name) +} diff --git a/acceptance/tests/openshift/main_test.go b/acceptance/tests/openshift/main_test.go new file mode 100644 index 0000000000..1167010fe3 --- /dev/null +++ b/acceptance/tests/openshift/main_test.go @@ -0,0 +1,23 @@ +package openshift + +import ( + "fmt" + "os" + "testing" + + testsuite "github.com/hashicorp/consul-k8s/acceptance/framework/suite" +) + +var suite testsuite.Suite + +func TestMain(m *testing.M) { + suite = testsuite.NewSuite(m) + + cfg := suite.Config() + if cfg.UseOpenshift { + os.Exit(suite.Run()) + } else { + fmt.Println("Skipping openshift tests because use-openshift not set") + os.Exit(0) + } +} diff --git a/control-plane/go.mod b/control-plane/go.mod index f1cb7dd378..3b09cd683f 100644 --- a/control-plane/go.mod +++ b/control-plane/go.mod @@ -37,7 +37,6 @@ require ( github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.25.0 golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 - golang.org/x/sync v0.8.0 golang.org/x/text v0.17.0 golang.org/x/time v0.3.0 gomodules.xyz/jsonpatch/v2 v2.4.0 @@ -147,6 +146,7 @@ require ( golang.org/x/mod v0.20.0 // indirect golang.org/x/net v0.28.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.24.0 // indirect golang.org/x/term v0.23.0 // indirect golang.org/x/tools v0.24.0 // indirect