chore: Automate functional tests in the pipeline

.github/workflows/functional.yml

@@ -0,0 +1,177 @@
+name: Functional Testing
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+*"
+  pull_request:
+  workflow_call:
+    inputs:
+      platforms:
+        required: true
+        type: string
+      image:
+        required: true
+        type: string
+      tag:
+        required: false
+        type: string
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    permissions:
+      contents: read # for docker/build-push-action to read repo content
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+      packages: write # for docker/build-push-action to push to GHCR
+      id-token: write # for docker/login to login to NGINX registry
+    runs-on: ubuntu-22.04
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
+    strategy:
+        matrix:
+          k8s-version: ["1.23.17", "latest"]
+          nginx-image: [nginx,nginx-plus]
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Setup Golang Environment
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        with:
+          go-version: stable
+
+      - name: Set GOPATH
+        run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+
+      - name: Docker Buildx
+        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0
+        with:
+          driver-opts: network=host
+
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+        with:
+          platforms: arm64
+
+      - name: Output Variables
+        id: vars
+        run: echo "go_path=$(go env GOPATH)" >> $GITHUB_OUTPUT
+
+      - name: Build binary
+        uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0
+        with:
+          version: latest
+          args: ${{ github.ref_type == 'tag' && 'release' || 'build --snapshot' }} --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GOPATH: ${{ steps.vars.outputs.go_path }}
+          AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }}
+          AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }}
+          AZURE_BUCKET_NAME: ${{ secrets.AZURE_BUCKET_NAME }}
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_COMMUNITY }}
+
+      - name: Fetch Cached Artifacts
+        uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
+        with:
+          path: ${{ github.workspace }}/dist
+          key: nginx-gateway-fabric-${{ github.run_id }}-${{ github.run_number }}
+
+      - name: NGF Docker meta
+        id: ngf-meta
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        with:
+          images: |
+            name=ghcr.io/nginxinc/nginx-gateway-fabric
+          tags: |
+            type=semver,pattern={{version}}
+            type=edge
+            type=ref,event=pr
+            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
+
+      - name: Update Go Modules
+        if: ${{ github.event_name == 'schedule' }}
+        run: make update-go-modules
+        working-directory: ./conformance
+
+      - name: NGINX Docker meta
+        id: nginx-meta
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        with:
+          images: |
+            name=ghcr.io/nginxinc/nginx-gateway-fabric/${{ matrix.nginx-image }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=edge
+            type=ref,event=pr
+            type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
+
+      - name: Build NGF Docker Image
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
+        with:
+          file: build/Dockerfile
+          tags: ${{ steps.ngf-meta.outputs.tags }}
+          context: "."
+          target: goreleaser
+          load: true
+          cache-from: type=gha,scope=ngf
+          cache-to: type=gha,scope=ngf,mode=max
+          pull: true
+
+      - name: Build NGINX Docker Image
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
+        with:
+          file: build/Dockerfile${{ matrix.nginx-image == 'nginx' && '.nginx' || '' }}${{ matrix.nginx-image == 'nginx-plus' && '.nginxplus' || ''}}
+          tags: ${{ steps.nginx-meta.outputs.tags }}
+          context: "."
+          load: true
+          cache-from: type=gha,scope=${{ matrix.nginx-image }}
+          cache-to: type=gha,scope=${{ matrix.nginx-image }},mode=max
+          pull: true
+          build-args: |
+            NJS_DIR=internal/mode/static/nginx/modules/src
+            NGINX_CONF_DIR=internal/mode/static/nginx/conf
+            BUILD_AGENT=gha
+          secrets: |
+            ${{ contains(inputs.image, 'plus') && format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) || '' }}
+            ${{ contains(inputs.image, 'plus') && format('"nginx-repo.key={0}"', secrets.NGINX_KEY) || '' }}
+
+      - name: Deploy Kubernetes
+        id: k8s
+        run: |
+          kube_config=${{ github.workspace }}/deploy/helm-chart/kube-${{ github.run_id }}-helm
+          make create-kind-cluster KIND_KUBE_CONFIG=${kube_config}
+          echo "KUBECONFIG=${kube_config}" >> "$GITHUB_ENV"
+
+      - name: Setup functional tests
+        run: |
+          ngf_tag=${{ steps.ngf-meta.outputs.version }}
+          if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi
+          if [ ${{ startsWith(matrix.k8s-version, '1.23') ||  startsWith(matrix.k8s-version, '1.24') }} == "true" ]; then export INSTALL_WEBHOOK=true; fi
+          make build-images${{ matrix.nginx-image == 'nginx-plus' && '-with-plus' || ''}} load-images${{ matrix.nginx-image == 'nginx-plus' && '-with-plus' || ''}} TAG=${ngf_tag}
+        working-directory: ./tests
+
+      - name: Run NGINX functional tests
+        run: |
+          ngf_tag=${{ steps.ngf-meta.outputs.version }}
+          if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi
+          make functional-test${{ matrix.nginx-image == 'nginx-plus' && '-nginx-plus' || ''}} TAG=${ngf_tag}
+        working-directory: ./tests
+
+      - name: Upload profile to release
+        if: ${{ matrix.k8s-version == 'latest' && startsWith(github.ref, 'refs/tags/') }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh release upload ${{ github.ref_name }} results/functional/edge/*
+        working-directory: ./tests

tests/Makefile

@@ -52,6 +52,14 @@ test: ## Run the system tests against your default k8s cluster
 		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --pull-policy=$(PULL_POLICY) \
 		--k8s-version=$(K8S_VERSION) --service-type=$(GW_SERVICE_TYPE) --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)
 
+.PHONY: functional-test
+functional-test: ## Run the functional tests with NGINX against your default k8s cluster
+	make test GINKGO_LABEL=functional
+
+.PHONY: functional-test-nginx-plus
+functional-test-nginx-plus: ## Run the functional tests with NGINX Plus against your default k8s cluster
+	make test GINKGO_LABEL=functional PLUS_ENABLED=true
+
 .PHONY: delete-kind-cluster
 delete-kind-cluster: ## Delete kind cluster
 	kind delete cluster

tests/README.md

@@ -37,6 +37,7 @@ make
 ```
 
 ```text
+build-images-with-plus         Build NGF and NGINX Plus images
 build-images                   Build NGF and NGINX images
 cleanup-gcp                    Cleanup all GCP resources
 cleanup-router                 Delete the GKE router
@@ -45,7 +46,10 @@ create-and-setup-vm            Create and setup a GCP VM for tests
 create-gke-router              Create a GKE router to allow egress traffic from private nodes (allows for external image pulls)
 create-kind-cluster            Create a kind cluster
 delete-kind-cluster            Delete kind cluster
+functional-test-nginx-plus     Run the functional tests with NGINX Plus against your default k8s cluster
+functional-test                Run the functional tests with NGINX against your default k8s cluster
 help                           Display this help
+load-images-with-plus          Load NGF and NGINX Plus images on configured kind cluster
 load-images                    Load NGF and NGINX images on configured kind cluster
 run-tests-on-vm                Run the tests on a GCP VM
 setup-gcp-and-run-tests        Create and setup a GKE router and GCP VM for tests and run the tests
@@ -101,7 +105,7 @@ make build-images-with-plus load-images-with-plus TAG=$(whoami)
 
 ## Step 3 - Run the tests
 
-### 3a - Run the tests locally
+### 3a - Run all the tests locally
 
 ```makefile
 make test TAG=$(whoami)
@@ -113,7 +117,19 @@ Or, to run the tests with NGINX Plus enabled:
 make test TAG=$(whoami) PLUS_ENABLED=true
 ```
 
-### 3b - Run the tests on a GKE cluster from a GCP VM
+### 3b - Run the functional tests locally
+
+```makefile
+make functional-test TAG=$(whoami)
+```
+
+Or, to run the tests with NGINX Plus enabled:
+
+```makefile
+make functional-test-nginx-plus TAG=$(whoami)
+```
+
+### 3c - Run the tests on a GKE cluster from a GCP VM
 
 This step only applies if you would like to run the tests from a GCP based VM.
 

tests/results/functional/edge/edge.md

@@ -0,0 +1,39 @@
+# Results
+
+## Test environment
+
+NGINX Plus: false
+
+Local Cluster:
+
+- Node count: 1
+- k8s version: v1.29.2
+- vCPUs per node: 12
+- RAM per node: 8034504Ki
+- Max pods per node: 110
+# Results
+
+## Test environment
+
+NGINX Plus: true
+
+Local Cluster:
+
+- Node count: 1
+- k8s version: v1.29.2
+- vCPUs per node: 12
+- RAM per node: 8034504Ki
+- Max pods per node: 110
+# Results
+
+## Test environment
+
+NGINX Plus: false
+
+Local Cluster:
+
+- Node count: 1
+- k8s version: v1.29.2
+- vCPUs per node: 12
+- RAM per node: 8034504Ki
+- Max pods per node: 110

tests/suite/sample_test.go

@@ -3,6 +3,8 @@ package suite
 import (
 	"fmt"
 	"net/http"
+	"os"
+	"path/filepath"
 	"strconv"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -14,7 +16,8 @@ import (
 	"github.com/nginxinc/nginx-gateway-fabric/tests/framework"
 )
 
-var _ = Describe("Basic test example", func() {
+var _ = Describe("Basic test example", Label("functional"), func() {
+	var outFile *os.File
 	files := []string{
 		"hello/hello.yaml",
 		"hello/gateway.yaml",
@@ -30,11 +33,20 @@ var _ = Describe("Basic test example", func() {
 		Expect(resourceManager.Apply([]client.Object{ns})).To(Succeed())
 		Expect(resourceManager.ApplyFromFiles(files, ns.Name)).To(Succeed())
 		Expect(resourceManager.WaitForAppsToBeReady(ns.Name)).To(Succeed())
+
+		resultsDir, err := framework.CreateResultsDir("functional", version)
+		Expect(err).ToNot(HaveOccurred())
+
+		filename := filepath.Join(resultsDir, fmt.Sprintf("%s.md", version))
+		outFile, err = framework.CreateResultsFile(filename)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(framework.WriteSystemInfoToFile(outFile, clusterInfo, *plusEnabled)).To(Succeed())
 	})
 
 	AfterEach(func() {
 		Expect(resourceManager.DeleteFromFiles(files, ns.Name)).To(Succeed())
 		Expect(resourceManager.Delete([]client.Object{ns})).To(Succeed())
+		outFile.Close()
 	})
 
 	It("sends traffic", func() {