Conformance Testing #2740
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Conformance Testing | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - release-* | |
| tags: | |
| - "v[0-9]+.[0-9]+.[0-9]+*" | |
| pull_request: | |
| schedule: | |
| - cron: "0 4 * * *" # run every day at 4am UTC | |
| defaults: | |
| run: | |
| shell: bash | |
| concurrency: | |
| group: ${{ github.ref_name }}-conformance | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| conformance-tests: | |
| name: Gateway Conformance Tests | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| k8s-version: ["1.23.17", "latest"] | |
| nginx-image: [nginx, nginx-plus] | |
| enable-experimental: [true, false] | |
| permissions: | |
| contents: write # needed for uploading release artifacts | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: Setup Golang Environment | |
| uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 | |
| with: | |
| go-version: stable | |
| - name: Set GOPATH | |
| run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV | |
| - name: Docker Buildx | |
| uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0 | |
| - name: NGF Docker meta | |
| id: ngf-meta | |
| uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 | |
| with: | |
| images: | | |
| name=ghcr.io/nginxinc/nginx-gateway-fabric | |
| tags: | | |
| type=semver,pattern={{version}} | |
| type=edge | |
| type=ref,event=pr | |
| type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} | |
| - name: NGINX Docker meta | |
| id: nginx-meta | |
| uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 | |
| with: | |
| images: | | |
| name=ghcr.io/nginxinc/nginx-gateway-fabric/${{ matrix.nginx-image }} | |
| tags: | | |
| type=semver,pattern={{version}} | |
| type=edge | |
| type=ref,event=pr | |
| type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} | |
| - name: Prepare NGF files | |
| run: | | |
| ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric | |
| ngf_tag=${{ steps.ngf-meta.outputs.version }} | |
| make update-ngf-manifest${{ matrix.nginx-image == 'nginx-plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} | |
| working-directory: ./conformance | |
| - name: Build binary | |
| uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0 | |
| with: | |
| version: latest | |
| args: build --snapshot --clean | |
| - name: Build NGF Docker Image | |
| uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0 | |
| with: | |
| file: build/Dockerfile | |
| tags: ${{ steps.ngf-meta.outputs.tags }} | |
| context: "." | |
| target: goreleaser | |
| load: true | |
| cache-from: type=gha,scope=ngf | |
| cache-to: type=gha,scope=ngf,mode=max | |
| pull: true | |
| - name: Build NGINX Docker Image | |
| uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0 | |
| with: | |
| file: build/Dockerfile${{ matrix.nginx-image == 'nginx' && '.nginx' || '' }}${{ matrix.nginx-image == 'nginx-plus' && '.nginxplus' || ''}} | |
| tags: ${{ steps.nginx-meta.outputs.tags }} | |
| context: "." | |
| load: true | |
| cache-from: type=gha,scope=${{ matrix.nginx-image }} | |
| cache-to: type=gha,scope=${{ matrix.nginx-image }},mode=max | |
| pull: true | |
| build-args: | | |
| NJS_DIR=internal/mode/static/nginx/modules/src | |
| NGINX_CONF_DIR=internal/mode/static/nginx/conf | |
| BUILD_AGENT=gha | |
| - name: Update Go Modules | |
| if: ${{ github.event_name == 'schedule' }} | |
| run: make update-go-modules | |
| working-directory: ./conformance | |
| - name: Build Test Docker Image | |
| uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0 | |
| with: | |
| file: conformance/tests/Dockerfile | |
| tags: conformance-test-runner:${{ github.sha }} | |
| context: "." | |
| load: true | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| pull: true | |
| - name: Deploy Kubernetes | |
| id: k8s | |
| run: | | |
| k8s_version=${{ matrix.k8s-version }} | |
| make create-kind-cluster KIND_KUBE_CONFIG=${{ github.workspace }}/kube-${{ github.run_id }} ${{ ! contains(matrix.k8s-version, 'latest') && 'KIND_IMAGE=kindest/node:v${k8s_version}' || '' }} | |
| echo "KUBECONFIG=${{ github.workspace }}/kube-${{ github.run_id }}" >> "$GITHUB_ENV" | |
| working-directory: ./conformance | |
| - name: Wait for release to exist | |
| if: startsWith(github.ref, 'refs/tags/') | |
| run: | | |
| REF=${{ github.ref_name }} | |
| until docker pull ghcr.io/nginxinc/nginx-gateway-fabric:${REF#v}; do sleep 5; done | |
| until docker pull ghcr.io/nginxinc/nginx-gateway-fabric/nginx:${REF#v}; do sleep 5; done | |
| - name: Setup conformance tests | |
| run: | | |
| ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric | |
| ngf_tag=${{ steps.ngf-meta.outputs.version }} | |
| if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi | |
| if [ ${{ startsWith(matrix.k8s-version, '1.23') || startsWith(matrix.k8s-version, '1.24') }} == "true" ]; then export INSTALL_WEBHOOK=true; fi | |
| if [ ${{ matrix.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi | |
| make install-ngf-local-no-build${{ matrix.nginx-image == 'nginx-plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} | |
| working-directory: ./conformance | |
| - name: Run conformance tests | |
| run: | | |
| make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} VERSION=${{ github.ref_name }} | |
| core_result=$(cat conformance-profile.yaml | yq '.profiles[0].core.result') | |
| extended_result=$(cat conformance-profile.yaml | yq '.profiles[0].extended.result') | |
| if [ "${core_result}" == "failure" ] || [ "${extended_result}" == "failure" ]; then echo "Conformance test failed, see above for details." && exit 2; fi | |
| working-directory: ./conformance | |
| - name: Upload profile to release | |
| if: ${{ matrix.k8s-version == 'latest' && startsWith(github.ref, 'refs/tags/') }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh release upload ${{ github.ref_name }} conformance-profile.yaml | |
| working-directory: ./conformance |