azure-pipelines.yml

# TODO: This pipeline is currently disabled. To enable it, go to https://dev.azure.com/azure/azure-service-operator/_build?definitionId=74 and enable the pipeline

# By default, ADO runs jobs on pushes and on PRs. Since we're only tracking/after PRs, turn off
# tiggers on push.
trigger:
  branches:
    # Run automatically ONLY on GitHub Merge Queue branches
    include: [ 'gh-readonly-queue/*' ]

# PR Section describes what happens on PR
pr:
  branches:
    include:
    - main

resources:
- repo: self

variables:
  tag: '$(Build.BuildId)'
  MAJOR_VERSION: 1
  MINOR_VERSION: 0
  PATCH_VERSION: $(Build.BuildId)
  IMAGE_NAME: 'candidate/k8s/azureserviceoperator'
  ASO_HELM_CHART_NAME: 'candidate/k8s/asohelmchart'
  KUBEBUILDER_ASSETS: $(go env GOPATH)/bin

  # Set these values in Azure Devops variables 

  # AKS_CLUSTER_RG: Name of Resource group where AKS clusters used for testing are present
  # OPERATOR_NAMESPACE: Namespace in AKS cluster that operator is deployed to (Default: azureoperator-system)
  # ASO-DEVOPS-MI: Name of Managed Identity to use to deploy the Operator as in AKS
  # POD-IDENTITY-CLIENTID: ClientID of Managed Identity to use to deploy the Operator as in AKS
  # AZURE_OPERATOR_KEYVAULT: Keyvault passed to Operator deployment to use for secrets
  # AZURE_CLIENT_ID: Client ID of Service Principal used to run integration tests on Kind Cluster, login to Helm registry ACR
  # AZURE_CLIENT_SECRET: Secret for the above Client ID
  # AZURE_SUBSCRIPTION_ID: Subscription ID that the pipeline runs against
  # AZURE_TENANT_ID: Tenant ID that the pipeline runs against 

  # Pipeline ACR - Temporary ACR used to hold the image until AKS deploy testing is done.
  # Release ACR - Only after the deploy test succeeds, the image and helm chart are pushed to the release ACR. 
  # The release pipeline exposes the images in Release ACR through mcr.microsoft.com

  # CONTAINER_REGISTRY: Service Connection Name for Release ACR.
  # CONTAINER_REGISTRY_NAME: Name of the Release ACR.
  # PIPELINE_CONTAINER_REGISTRY: Service Connection Name for temporary ACR used in pipeline. 
  # PIPELINE_CONTAINER_REGISTRY_NAME: Name of the temporary ACR used in the pipeline.

  # TEST_APIM_NAME: Used in integration tests - Name of the APIM service that is pre-configured.
  # TEST_APIM_RG: Used in integration tests - Name of the resource group the pre-configured APIM service is in.

  # Agent VM image name
  vmImageName: 'ubuntu-latest'

jobs:
- job: Build
  timeoutInMinutes: 90
  pool:
    vmImage: 'ubuntu-latest'
  steps:
    - checkout: self  # self represents the repo where the initial Pipelines YAML file was found
      lfs: "true"

    - task: Bash@3
      name: "check_changes"
      inputs:
        targetType: 'filePath'
        filePath: './scripts/check-changes.sh'
      
    - script: |
        make validate-copyright-headers
        make validate-cainjection-files
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      displayName: 'Validate Source'
      continueOnError: 'false'

    # TODO: Tried to update to Go 1.22 but failed, see https://github.com/microsoft/azure-pipelines-tasks/issues/19690
    - task: GoTool@0
      displayName: Get Go 1.20
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      inputs:
        version: '1.20'

    - task: Go@0
      displayName: Get Go dependencies
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      inputs:
        command: 'get'
        arguments: '-d'
        workingDirectory: '$(System.DefaultWorkingDirectory)'

    - task: Bash@3
      displayName: Install kubebuilder, kustomize and test dependencies
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      continueOnError: 'false'
      inputs:
        targetType: 'inline'
        script: |
          set -x -e
          mkdir -p $(System.DefaultWorkingDirectory)/bin
          go env
          os=$(go env GOOS)
          arch=$(go env GOARCH)
          go mod download
          make install-kubebuilder
          make install-test-tools
          make generate-test-certs
        workingDirectory: '$(System.DefaultWorkingDirectory)'

    - script: |
        set -e
        export PATH=$PATH:$(go env GOPATH)/bin
        make test-unit
      displayName: Run unit tests
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      continueOnError: 'false'
      env:
        GO111MODULE: on
        BUILD_ID: $(Build.BuildId)
      workingDirectory: '$(System.DefaultWorkingDirectory)'

    - script: |
        set -e
        export PATH=$PATH:$(go env GOPATH)/bin:$(go env GOPATH)/kubebuilder/bin
        export KUBEBUILDER_ASSETS=$(go env GOPATH)/kubebuilder/bin
        export TEST_APIM_RG=$(TEST_APIM_RG)
        export TEST_APIM_NAME=$(TEST_APIM_NAME)
        make test-integration-controllers
      displayName: Run envtest tests
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      continueOnError: 'false'
      env:
        GO111MODULE: on
        AZURE_SUBSCRIPTION_ID: $(AZURE_SUBSCRIPTION_ID)
        AZURE_TENANT_ID: $(AZURE_TENANT_ID)
        AZURE_CLIENT_ID: $(AZURE_CLIENT_ID)
        AZURE_CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
        REQUEUE_AFTER: $(REQUEUE_AFTER)
        BUILD_ID: $(Build.BuildId)
      workingDirectory: '$(System.DefaultWorkingDirectory)'

    # TODO: There is no way to run steps in parallel in Azure pipelines but ideally this step would run in parallel
    # TODO: with the above testing step to reduce overall runtime
    - script: |
        set -e
        export PATH=$PATH:$(go env GOPATH)/bin:$(go env GOPATH)/kubebuilder/bin
        export KUBEBUILDER_ASSETS=$(go env GOPATH)/kubebuilder/bin
        make test-v1-secret-naming
      displayName: Run legacy v1 secret naming tests
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      continueOnError: 'false'
      env:
        GO111MODULE: on
        AZURE_SUBSCRIPTION_ID: $(AZURE_SUBSCRIPTION_ID)
        AZURE_TENANT_ID: $(AZURE_TENANT_ID)
        AZURE_CLIENT_ID: $(AZURE_CLIENT_ID)
        AZURE_CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
        REQUEUE_AFTER: $(REQUEUE_AFTER)
        BUILD_ID: $(Build.BuildId)
      workingDirectory: '$(System.DefaultWorkingDirectory)'

    # TODO: There is no way to run steps in parallel in Azure pipelines but ideally this step would run in parallel
    # TODO: with the above testing step to reduce overall runtime
    - script: |
        set -e
        export PATH=$PATH:$(go env GOPATH)/bin:$(go env GOPATH)/kubebuilder/bin
        export KUBEBUILDER_ASSETS=$(go env GOPATH)/kubebuilder/bin
        make test-no-target-namespaces
      displayName: Run test for no target namespaces
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      continueOnError: 'false'
      env:
        GO111MODULE: on
        AZURE_SUBSCRIPTION_ID: $(AZURE_SUBSCRIPTION_ID)
        AZURE_TENANT_ID: $(AZURE_TENANT_ID)
        AZURE_CLIENT_ID: $(AZURE_CLIENT_ID)
        AZURE_CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
        REQUEUE_AFTER: $(REQUEUE_AFTER)
        BUILD_ID: $(Build.BuildId)
      workingDirectory: '$(System.DefaultWorkingDirectory)'

    # TODO: There is no way to run steps in parallel in Azure pipelines but ideally this step would run in parallel
    # TODO: with the above testing step to reduce overall runtime
    - script: |
        set -e
        export PATH=$PATH:$(go env GOPATH)/bin:$(go env GOPATH)/kubebuilder/bin
        export KUBEBUILDER_ASSETS=$(go env GOPATH)/kubebuilder/bin
        make test-webhooks-only-mode
      displayName: Run webhooks-only mode tests
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      continueOnError: 'false'
      env:
        GO111MODULE: on
        AZURE_SUBSCRIPTION_ID: $(AZURE_SUBSCRIPTION_ID)
        AZURE_TENANT_ID: $(AZURE_TENANT_ID)
        AZURE_CLIENT_ID: $(AZURE_CLIENT_ID)
        AZURE_CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
        REQUEUE_AFTER: $(REQUEUE_AFTER)
        BUILD_ID: $(Build.BuildId)
      workingDirectory: '$(System.DefaultWorkingDirectory)'

    # TODO: There is no way to run steps in parallel in Azure pipelines but ideally this step would run in parallel
    # TODO: with the above testing step to reduce overall runtime
    - script: |
        set -e
        export PATH=$PATH:$(go env GOPATH)/bin:$(go env GOPATH)/kubebuilder/bin
        export KUBEBUILDER_ASSETS=$(go env GOPATH)/kubebuilder/bin
        make test-watchers-only-mode
      displayName: Run watchers-only mode tests
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      continueOnError: 'false'
      env:
        GO111MODULE: on
        AZURE_SUBSCRIPTION_ID: $(AZURE_SUBSCRIPTION_ID)
        AZURE_TENANT_ID: $(AZURE_TENANT_ID)
        AZURE_CLIENT_ID: $(AZURE_CLIENT_ID)
        AZURE_CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
        REQUEUE_AFTER: $(REQUEUE_AFTER)
        BUILD_ID: $(Build.BuildId)
      workingDirectory: '$(System.DefaultWorkingDirectory)'

    - script: |
        set -e
        export PATH=$PATH:$(go env GOPATH)/bin
        make kind-create
        kubectl cluster-info
        make install
      displayName: Install CRDs into kind cluster
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      continueOnError: 'false'
      env:
        GO111MODULE: on
        KUBEBUILDER_ASSETS: $(go env GOPATH)/bin
        BUILD_ID: $(Build.BuildId)
      workingDirectory: '$(System.DefaultWorkingDirectory)'

    - bash: |
        export PATH=$PATH:$(go env GOPATH)/bin
        make test-process-coverage
      displayName: Render Coverage Reports
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      continueOnError: true
      workingDirectory: '$(System.DefaultWorkingDirectory)'
      
    - task: PublishCodeCoverageResults@1
      displayName: Publish Codecoverage results
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      inputs:
        codeCoverageTool: Cobertura 
        summaryFileLocation: $(System.DefaultWorkingDirectory)/reports/merged-coverage.xml
    
    - task: PublishBuildArtifacts@1
      displayName: Publish test reports
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      inputs:
        pathtoPublish: '$(System.DefaultWorkingDirectory)/reports'
        artifactName: reports
    
    - task: AzureCLI@2
      displayName: Test - Cleanup Azure Resources
      condition: always()
      inputs:
        azureSubscription: 'ASO Subscription 3'
        scriptType: 'bash'
        scriptLocation: 'inlineScript'
        inlineScript: 'make test-cleanup-azure-resources'

    - script: docker build -t $(IMAGE_NAME) .
      displayName: Docker build
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      workingDirectory: '$(System.DefaultWorkingDirectory)'

    - task: Docker@2
      displayName: Login to temporary pipeline ACR
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      inputs:
        containerRegistry: $(PIPELINE_CONTAINER_REGISTRY)
        command: 'login'

    - task: Docker@2
      displayName: Build and Push Docker Image to temporary ACR for validation
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      inputs:
        containerRegistry: $(PIPELINE_CONTAINER_REGISTRY)
        repository: '$(IMAGE_NAME)'
        command: 'buildAndPush'
        Dockerfile: '$(System.DefaultWorkingDirectory)/Dockerfile'
        buildContext: '$(System.DefaultWorkingDirectory)'
        tags: '$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION)'
    
    - task: AzureCLI@2
      inputs:
        azureSubscription: 'ASO Subscription 3'
        scriptType: 'bash'
        scriptLocation: 'inlineScript'
        inlineScript: |
          az resource list --tag freeforpipeline=true --query "[?(type == 'Microsoft.ContainerService/managedClusters') || (type == 'microsoft.containerservice/managedclusters')].{Name:name}" -o table > clusters.txt
          clustername=$(tail -n 1 clusters.txt)
          if [ "$clustername" == "" ]; then
            echo AKS clusters unavailable
            exit 1
          fi
          echo $clustername
          echo "##vso[task.setvariable variable=chosenclustername]$clustername"
          echo 'az aks get-credentials -g $(AKS_CLUSTER_RG) -n $clustername'
          az aks get-credentials -g $(AKS_CLUSTER_RG) -n $clustername
          # Set tags to not available for the selected cluster so it doesn't get used in another run
          az resource tag --tags 'freeforpipeline=false' -g $(AKS_CLUSTER_RG) -n $clustername --resource-type Microsoft.ContainerService/managedClusters
        workingDirectory: '$(System.DefaultWorkingDirectory)'
      displayName: Deploy to AKS - Find available AKS cluster and connect to it
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))

    - script: |
        kubectl create namespace cert-manager
        kubectl label namespace cert-manager cert-manager.io/disable-validation=true
        kubectl apply --validate=false -f https://github.com/cert-manager/cert-manager/releases/download/v1.7.2/cert-manager.yaml
      displayName: Deploy to AKS - Install Cert Manager
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))

    - task: HelmInstaller@1
      displayName: Deploy to AKS - Install latest Helm
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      inputs:
        helmVersionToInstall: 3.6.3  # Helm 3.7 renamed "hem chart" commands, which we use
    - script: |
        # Replace image repository in values.yaml to the official image in ACR
        img="$MAJOR_VERSION.$MINOR_VERSION.$PATCH_VERSION"
        echo $img
        sed -i -e 's@azureserviceoperator:.*$@azureserviceoperator:'${img}'@' charts/azure-service-operator/values.yaml
      displayName: Deploy to AKS - Replace image in values.yaml

    - task: Bash@3
      displayName: Deploy to AKS - Helm Deploy
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      inputs:
        targetType: 'inline'
        script: |
          set -euxo pipefail
          kubectl delete namespace --ignore-not-found $(OPERATOR_NAMESPACE)
          imagename="$(PIPELINE_CONTAINER_REGISTRY_NAME)/$(IMAGE_NAME):$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION)"
          # Apply CRDs as Helm 3 does not install CRDs for Helm upgrade
          kubectl apply -f ./charts/azure-service-operator/crds/
          # Find the latest ASO chart version
          latestchart=$($(go env GOPATH)/bin/yq eval '.entries.azure-service-operator[] | .version' charts/index.yaml | sort | tail -1)
          # Deploy using Helm
          helm upgrade --install aso charts/azure-service-operator-${latestchart}.tgz -n $(OPERATOR_NAMESPACE) --create-namespace \
              --set azureSubscriptionID=$(AZURE_SUBSCRIPTION_ID) \
              --set azureTenantID=$(AZURE_TENANT_ID) \
              --set azureClientID=$(AZURE_CLIENT_ID) \
              --set azureClientSecret=$(AZURE_CLIENT_SECRET) \
              --set azureUseMI=true \
              --set installAadPodIdentity=true \
              --set azureOperatorKeyvault=$(AZURE_OPERATOR_KEYVAULT) \
              --set aad-pod-identity.azureIdentities.aso-identity.resourceID="/subscriptions/$(AZURE_SUBSCRIPTION_ID)/resourcegroups/resourcegroup-azure-operators/providers/Microsoft.ManagedIdentity/userAssignedIdentities/$(ASO-DEVOPS-MI)" \
              --set aad-pod-identity.azureIdentities.aso-identity.clientID=$(POD-IDENTITY-CLIENTID) \
              --set aad-pod-identity.nmi.allowNetworkPluginKubenet=true \
              --set image.repository=$imagename \
              --set certManagerResourcesAPIVersion=cert-manager.io/v1
          # Verify namespace and pods
          kubectl get namespace
          kubectl get pods -n $(OPERATOR_NAMESPACE)
          kubectl describe pods -n $(OPERATOR_NAMESPACE)
          kubectl rollout status deployment/azureoperator-controller-manager -n $(OPERATOR_NAMESPACE) --timeout 120s
          kubectl logs -n $(OPERATOR_NAMESPACE) deployment/azureoperator-controller-manager -c manager
    
    - task: AzureCLI@2
      displayName: Deploy to AKS - Clean up deployment and release cluster back to free pool
      condition: or(eq(variables['check_changes.SOURCE_CODE_CHANGED'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      inputs:
        azureSubscription: 'ASO Subscription 3'
        scriptType: 'bash'
        scriptLocation: 'inlineScript'
        inlineScript: |
          set -e
          echo "Chosen AKS Cluster name"
          echo $(chosenclustername)
          # Delete CRDs to clean up cluster
          kubectl delete --ignore-not-found -f ./charts/azure-service-operator/crds/
          # Remove Helm deployment
          helm list -n $(OPERATOR_NAMESPACE)
          helm delete aso -n $(OPERATOR_NAMESPACE)
          # Set tags to available for the selected cluster to put it back into the free pool
          echo "Setting tags back to free"
          az resource tag --tags 'freeforpipeline=true' -g $(AKS_CLUSTER_RG) -n $(chosenclustername) --resource-type Microsoft.ContainerService/managedClusters
        workingDirectory: '$(System.DefaultWorkingDirectory)'
        # Turn off this check until our aad-pod-identity dep is updated
        # so that it's not trying to install v1beta1
        # ClusterRoleBindings.
      

    - task: Docker@2
      condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      displayName: Login to release ACR
      inputs:
        containerRegistry: $(CONTAINER_REGISTRY)
        command: 'login'

    - task: Docker@2
      condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      displayName: Build and Push Docker Image $(IMAGE_NAME)
      inputs:
        containerRegistry: $(CONTAINER_REGISTRY)
        repository: '$(IMAGE_NAME)'
        command: 'buildAndPush'
        Dockerfile: '$(System.DefaultWorkingDirectory)/Dockerfile'
        buildContext: '$(System.DefaultWorkingDirectory)'
        tags: |
          latest
          $(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION)
    
    - script: |
        set -e
        export HELM_EXPERIMENTAL_OCI=1
        cd charts/azure-service-operator
        versiontag="$(CONTAINER_REGISTRY_NAME)/$(ASO_HELM_CHART_NAME):$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION)"
        latesttag="$(CONTAINER_REGISTRY_NAME)/$(ASO_HELM_CHART_NAME):latest"
        echo $versiontag
        echo $latesttag
        echo $(AZURE_CLIENT_SECRET)|helm registry login $(CONTAINER_REGISTRY_NAME) --username $(AZURE_CLIENT_ID) --password-stdin
        helm chart save . $versiontag
        helm chart save . $latesttag
        helm chart push $versiontag
        helm chart push $latesttag
      continueOnError: 'true'
      displayName: Push Helm chart to MCR repo
      workingDirectory: '$(System.DefaultWorkingDirectory)'
      condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))

    - script: |
        set -e
        echo "updating kustomize image patch file for manager resource"
        echo $IMAGE_NAME
        img="$IMAGE_NAME:$MAJOR_VERSION.$MINOR_VERSION.$PATCH_VERSION"
        echo $img
        sed -i -e 's@controller:latest@'${img}'@g' ./config/default/manager_image_patch.yaml
        echo "GOPATH is: $(go env GOPATH)"
        echo 
        ls -l $(go env GOPATH)
        ls -l $(go env GOPATH)/bin
        which $(go env GOPATH)/bin/kustomize
        $(go env GOPATH)/bin/kustomize version
        $(go env GOPATH)/bin/kustomize build config/default > $(Build.ArtifactStagingDirectory)/setup.yaml
        set -x
        echo $img > $(Build.ArtifactStagingDirectory)/azure-service-operator.txt
        mkdir $(Build.ArtifactStagingDirectory)/scripts
        cp devops/release/scripts/*.sh $(Build.ArtifactStagingDirectory)/scripts
      continueOnError: 'false'
      displayName: Package artifacts (image, setup.yaml) for release
      workingDirectory: '$(System.DefaultWorkingDirectory)'
      condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) 

    - task: PublishBuildArtifacts@1
      condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
      inputs:
        pathtoPublish: $(Build.ArtifactStagingDirectory)
        artifactName: drop