Merge pull request #1656 from aztfmod/int-5.7.0

5.7.0
aztfmod · Jun 15, 2023 · b0665d3 · b0665d3
2 parents 830c9ac + 2d04d9a
commit b0665d3
Show file tree

Hide file tree

Showing 437 changed files with 8,672 additions and 1,397 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -26,15 +26,16 @@
     // "shutdownAction": "none",
 
     // Uncomment the next line to run commands after the container is created.
-    "postCreateCommand": "sudo cp -R /tmp/.ssh-localhost/* ~/.ssh && sudo chown -R $(whoami):$(whoami) /tf/caf ~/.ssh && sudo chmod 400 ~/.ssh/* && git config --global core.editor vi && pre-commit install && pre-commit autoupdate",
+	"postCreateCommand": "sudo cp -R /tmp/.ssh-localhost/* ~/.ssh && sudo chown -R $(whoami):$(whoami) /tf/caf ~/.ssh && sudo chmod 400 ~/.ssh/* && git config --global core.editor vi && pre-commit install && pre-commit autoupdate",
 
     // Add the IDs of extensions you want installed when the container is created in the array below.
     "customizations": {
         "vscode": {
             "extensions": [
 				"hashicorp.terraform",
 				"mutantdino.resourcemonitor",
-				"eamodio.gitlens"
+				"eamodio.gitlens",
+				"GitHub.copilot"
 			]
         }
     }

diff --git a/.github/workflows/developer.md b/.github/workflows/developer.md
@@ -0,0 +1,27 @@
+
+You can test the workflows on your local machine
+
+## MacOS
+
+Install act to run Github workflows on your local machine
+```
+brew install act
+```
+
+from the root of the module
+
+```
+# Make sure you have a GH_TOKEN with repo and workflow privileges
+export GH_TOKEN=xxx
+
+act --container-architecture linux/arm64 \
+  -s GITHUB_TOKEN=$GITHUB_TOKEN \
+  -s ARM_CLIENT_ID=$ARM_CLIENT_ID \
+  -s ARM_CLIENT_SECRET=$ARM_CLIENT_SECRET \
+  -s ARM_TENANT_ID=$ARM_TENANT_ID  \
+  -s ARM_SUBSCRIPTION_ID=$ARM_SUBSCRIPTION_ID \
+  -P ubuntu-latest=catthehacker/ubuntu:act-latest \
+  -W .github/workflows/landingzone-scenarios.yaml \
+  --input scenario=landingzone-scenarios-longrunners.json \
+  workflow_dispatch
+```
diff --git a/.github/workflows/landingzone-scenarios-longrunners.json b/.github/workflows/landingzone-scenarios-longrunners.json
@@ -14,7 +14,17 @@
           "name": "sqlmi2",
           "path": "mssql_mi/200-mi-native-two-regions/landingzone/phase2/sqlmi2"
         }
+      ],
+      "phase3": [
+        {
+          "name": "sqlmisecu",
+          "path": "mssql_mi/200-mi-native-two-regions/landingzone/phase3/sqlmisecu"
+        },
+        {
+          "name": "databases",
+          "path": "mssql_mi/200-mi-native-two-regions/landingzone/phase3/databases"
+        }
       ]
     }
   ]
-}
+}
diff --git a/.github/workflows/landingzone-scenarios.yaml b/.github/workflows/landingzone-scenarios.yaml
@@ -8,6 +8,22 @@ name: landingzone
 on:
   workflow_dispatch:
     inputs:
+      destroy:
+        description: Destroy the deployments at the end.
+        default: true
+        type: bool
+      landingzone_ref:
+        description: Set the base terraform landingzone tag, branch or ref to use to deploy the code
+        default: sqlmi.native.bugbash2
+        type: string
+      restart_phase:
+        description: "Select the phase to restart the job from:"
+        required: true
+        type: choice
+        default: phase1
+        options:
+          - phase1
+          - phase2
       scenario:
         description: "Select the scenario you want to run:"
         required: false
@@ -21,7 +37,7 @@ env:
   TF_CLI_ARGS_destroy: "-auto-approve -refresh=false"
   TF_REGISTRY_DISCOVERY_RETRY: 5
   TF_REGISTRY_CLIENT_TIMEOUT: 15
-  LZ_REF: int-5.7.0
+  LZ_REF: ${{ inputs.landingzone_ref }}
   ROVER_RUNNER: true
 
 jobs:
@@ -36,29 +52,41 @@ jobs:
           jobs=$( cat ./.github/workflows/${{ github.event.inputs.scenario }} | jq -c .)
           echo "jobs=${jobs}" >> $GITHUB_OUTPUT
 
-
-
-  job:
+  deploy:
     name: ${{ matrix.jobs.name }}
     needs: load_scenarios
 
     strategy:
+      max-parallel: 20
       matrix: ${{ fromJSON(needs.load_scenarios.outputs.jobs) }}
 
     uses: ./.github/workflows/phases.yaml
     secrets: inherit
     with:
       matrix: ${{ toJSON(matrix.jobs) }}
 
+  destroy:
+    name: ${{ matrix.jobs.name }} (destroy)
+    needs: [load_scenarios, deploy]
+
+    strategy:
+      max-parallel: 20
+      matrix: ${{ fromJSON(needs.load_scenarios.outputs.jobs) }}
+
+    uses: ./.github/workflows/phases_destroy.yaml
+    secrets: inherit
+    with:
+      matrix: ${{ toJSON(matrix.jobs) }}
+
   purge:
     name: purge
     runs-on: ubuntu-latest
-    if: ${{ failure() || cancelled() }}
+    if: inputs.destroy && (failure() || cancelled())
 
     needs: [job]
 
     container:
-      image: aztfmod/rover:1.4.6-2305.0405
+      image: aztfmod/rover:1.4.6-2305.1701
       options: --user 0
 
     steps:

diff --git a/.github/workflows/phases.yaml b/.github/workflows/phases.yaml
@@ -1,5 +1,5 @@
 
-name: job_phases
+name: phases_dispatcher
 
 on:
   workflow_call:
@@ -17,6 +17,7 @@ jobs:
       name: ${{ steps.setup.outputs.name }}
       phase1: ${{ steps.setup.outputs.phase1 }}
       phase2: ${{ steps.setup.outputs.phase2 }}
+      phase3: ${{ steps.setup.outputs.phase3 }}
       lz_ref: ${{ steps.setup.outputs.lz_ref }}
     steps:
       - name: set outputs with default values
@@ -27,18 +28,22 @@ jobs:
           name=$(echo '${{inputs.matrix}}' | jq -c .name) && echo "name: $name"
           phase1=$(echo '${{inputs.matrix}}' | jq -c .phase1) && echo "phase1: $phase1"
           phase2=$(echo '${{inputs.matrix}}' | jq -c .phase2) && echo "phase2: $phase2"
+          phase3=$(echo '${{inputs.matrix}}' | jq -c .phase3) && echo "phase3: $phase3"
           lz_ref=$(echo '${{inputs.matrix}}' | jq -c .lz_ref) && echo "lz_ref: $lz_ref"
           #
           echo "name=$name" >> $GITHUB_OUTPUT
           echo "phase1=$phase1" >> $GITHUB_OUTPUT
           echo "phase2=$phase2" >> $GITHUB_OUTPUT
+          echo "phase3=$phase3" >> $GITHUB_OUTPUT
           echo "lz_ref=$lz_ref" >> $GITHUB_OUTPUT
 
 
   phase1_deploy:
     needs: setup
 
     strategy:
+      max-parallel: 20
+      fail-fast: false
       matrix:
         phase: ${{ fromJSON(needs.setup.outputs.phase1) }}
 
@@ -54,6 +59,8 @@ jobs:
     needs: [setup, phase1_deploy]
 
     strategy:
+      max-parallel: 20
+      fail-fast: false
       matrix:
         phase: ${{ fromJSON(needs.setup.outputs.phase2) }}
 
@@ -65,3 +72,21 @@ jobs:
       phase: ${{ toJSON(matrix.phase) }}
       lz_ref: ${{ fromJSON(needs.setup.outputs.lz_ref) }}
 
+
+  phase3_deploy:
+    needs: [setup, phase2_deploy]
+
+    strategy:
+      max-parallel: 20
+      fail-fast: false
+      matrix:
+        phase: ${{ fromJSON(needs.setup.outputs.phase3) }}
+
+    uses: ./.github/workflows/rover.yaml
+    secrets: inherit
+    with:
+      scenario: ${{ needs.setup.outputs.name }}
+      name: phase3
+      phase: ${{ toJSON(matrix.phase) }}
+      lz_ref: ${{ fromJSON(needs.setup.outputs.lz_ref) }}
+
diff --git a/.github/workflows/phases_destroy.yaml b/.github/workflows/phases_destroy.yaml
@@ -0,0 +1,94 @@
+
+name: phases_dispatcher_destroy
+
+on:
+  workflow_call:
+    inputs:
+      matrix:
+        required: true
+        type: string
+
+jobs:
+
+  setup_destroy:
+    name: setup phases
+    runs-on: ubuntu-latest
+    outputs:
+      name: ${{ steps.setup_destroy.outputs.name }}
+      phase1: ${{ steps.setup_destroy.outputs.phase1 }}
+      phase2: ${{ steps.setup_destroy.outputs.phase2 }}
+      phase3: ${{ steps.setup_destroy.outputs.phase3 }}
+      lz_ref: ${{ steps.setup_destroy.outputs.lz_ref }}
+    steps:
+      - name: set outputs with default values
+        id: setup_destroy
+        run: |
+          echo '${{inputs.matrix}}' | jq -c
+          #
+          name=$(echo '${{inputs.matrix}}' | jq -c .name) && echo "name: $name"
+          phase1=$(echo '${{inputs.matrix}}' | jq -c .phase1) && echo "phase1: $phase1"
+          phase2=$(echo '${{inputs.matrix}}' | jq -c .phase2) && echo "phase2: $phase2"
+          phase3=$(echo '${{inputs.matrix}}' | jq -c .phase3) && echo "phase3: $phase3"
+          lz_ref=$(echo '${{inputs.matrix}}' | jq -c .lz_ref) && echo "lz_ref: $lz_ref"
+          #
+          echo "name=$name" >> $GITHUB_OUTPUT
+          echo "phase1=$phase1" >> $GITHUB_OUTPUT
+          echo "phase2=$phase2" >> $GITHUB_OUTPUT
+          echo "phase3=$phase3" >> $GITHUB_OUTPUT
+          echo "lz_ref=$lz_ref" >> $GITHUB_OUTPUT
+
+
+  phase3_destroy:
+    needs: [setup_destroy]
+
+    strategy:
+      max-parallel: 20
+      fail-fast: false
+      matrix:
+        phase: ${{ fromJSON(needs.setup_destroy.outputs.phase3) }}
+
+    uses: ./.github/workflows/rover.yaml
+    secrets: inherit
+    with:
+      scenario: ${{ needs.setup_destroy.outputs.name }}
+      name: phase3
+      phase: ${{ toJSON(matrix.phase) }}
+      lz_ref: ${{ fromJSON(needs.setup_destroy.outputs.lz_ref) }}
+      plan_options: -destroy
+
+  phase2_destroy:
+    needs: [setup_destroy, phase3_destroy]
+
+    strategy:
+      max-parallel: 20
+      fail-fast: false
+      matrix:
+        phase: ${{ fromJSON(needs.setup_destroy.outputs.phase2) }}
+
+    uses: ./.github/workflows/rover.yaml
+    secrets: inherit
+    with:
+      scenario: ${{ needs.setup_destroy.outputs.name }}
+      name: phase2
+      phase: ${{ toJSON(matrix.phase) }}
+      lz_ref: ${{ fromJSON(needs.setup_destroy.outputs.lz_ref) }}
+      plan_options: -destroy
+
+  phase1_destroy:
+    needs: [setup_destroy, phase2_destroy]
+
+    strategy:
+      max-parallel: 20
+      fail-fast: false
+      matrix:
+        phase: ${{ fromJSON(needs.setup_destroy.outputs.phase1) }}
+
+    uses: ./.github/workflows/rover.yaml
+    secrets: inherit
+    with:
+      scenario: ${{ needs.setup_destroy.outputs.name }}
+      name: phase1
+      phase: ${{ toJSON(matrix.phase) }}
+      lz_ref: ${{ fromJSON(needs.setup_destroy.outputs.lz_ref) }}
+      plan_options: -destroy
+
diff --git a/.github/workflows/rover.yaml b/.github/workflows/rover.yaml
@@ -15,6 +15,9 @@ on:
       lz_ref:
         required: true
         type: string
+      plan_options:
+        required: false
+        type: string
 
 jobs:
 
@@ -48,7 +51,11 @@ jobs:
 
     env:
       ROVER_RUNNER: true
+      ARM_USE_AZUREAD: true
       ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+      ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
+      ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+      ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
 
     steps:
       - uses: actions/checkout@v3
@@ -77,7 +84,7 @@ jobs:
           #
           /tf/rover/rover.sh init
 
-      - name: Terraform Plan
+      - name: Terraform Plan ${{ inputs.plan_options }}
         id: tf_plan
         run: |
           /tf/rover/rover.sh \
@@ -87,10 +94,10 @@ jobs:
             -p ${{ env.PLAN_FILE }} \
             -var tags='{testing_job_id='"${{ github.run_id }}"'}' \
             -var var_folder_path=${{ env.CURRENT_FOLDER }} \
-            -a plan
+            -a plan ${{ inputs.plan_options }}
 
 
-      - name: Terraform Apply
+      - name: Terraform Apply ${{ inputs.plan_options }}
         id: tf_apply
         run: |
           /tf/rover/rover.sh \

diff --git a/.github/workflows/standalone-regressor-tf100.yaml b/.github/workflows/standalone-regressor-tf100.yaml
@@ -18,6 +18,7 @@ on:
         type: choice
         default: 'standalone-scenarios.json'
         options:
+        - standalone-scenarios-azuread.json
         - standalone-scenarios.json
         - standalone-compute.json
         - standalone-networking.json

diff --git a/.github/workflows/standalone-scenarios-azuread.json b/.github/workflows/standalone-scenarios-azuread.json
@@ -0,0 +1,16 @@
+{
+  "config_files": [
+    "azuread/100-azuread_administrative_units",
+    "azuread/100-azuread-application-with-sevice-principle-with-builtin-roles",
+    "azuread/100-sevice-principle-with-builtin-roles",
+    "azuread/101-azuread_administrative_unit_member",
+    "azuread/101-azuread-application-with-service-principle-with-custom-roles",
+    "azuread/101-service-principle-with-custom-roles",
+    "azuread/102-password-rotation",
+    "azuread/103-service-principal-only",
+    "azuread/104-azuread-group-membership",
+    "azuread/105-azuread-application-with-optional-claims",
+    "azuread/106-azuread-application-with-api-scopes",
+    "azuread/201-groups-and-roles"
+  ]
+}
diff --git a/.github/workflows/standalone-scenarios-longrunners.json b/.github/workflows/standalone-scenarios-longrunners.json
@@ -19,7 +19,6 @@
     "apim/117-api_management_product",
     "app_gateway/301-agw-v1",
     "compute/vmware_cluster/101-vmware_cluster",
-    "mssql_mi/200-mi",
     "networking/virtual_network_gateway/100-expressroute-gateway",
     "networking/virtual_network_gateway/101-vpn-site-to-site",
     "networking/virtual_network_gateway/102-vpn-site-to-site-active-active",