test → val (26 July 2024)

.github/audit-account.sh

@@ -4,32 +4,33 @@ git fetch --all > /dev/null
 
 #Parse inputs
 case ${1-} in
-  "ci_active"|"ci_inactive"|"cf_other"|"untagged")
+  "ci_active"|"ci_inactive"|"cf_other"|"untagged"|"orphaned_topics")
     OP=${1-}
     ;;
   *)
     echo "Error:  unkown operation"
-    echo "Usage: ${0} [ci_active|ci_inactive|cf_other|untagged] [resource_tagging_response|null]" && exit 1
+    echo "Usage: ${0} [ci_active|ci_inactive|cf_other|untagged|orphaned_topics] [resource_tagging_response|null]" && exit 1
     ;;
 esac
 
 shift
 if [ ! -z "${1-}" ]; then
   if [ -f "${1-}" ]; then
     RESOURCES=$(<"${1-}")
-  else 
+  else
     RESOURCES="${@-}"  
   fi
   jq empty <<< "${RESOURCES}"
   [ "$?" != 0 ] && echo "Error:  supplied JSON is invalid." && echo ${RESOURCES} && exit 1  
 else
+  export REGION=us-east-1
   RESOURCES=$(aws resourcegroupstaggingapi get-resources)
 fi
 
 #Create array of objects with the branch name and the interpolated branch name (for bot created branches)
 get_branches () {
- RAW_BRANCHES=$(git for-each-ref --format='%(refname)' refs/remotes/origin | sed 's|^.\+\/||g')
- BRANCHES=()
+ local RAW_BRANCHES=$(git for-each-ref --format='%(refname)' refs/remotes/origin | sed 's|^.\+\/||g')
+ local BRANCHES=()
  for B in $RAW_BRANCHES; do
    [ "${B}" == "HEAD" ] && continue
    IBRANCH=$(./setBranchName.sh ${B})
@@ -73,5 +74,28 @@ untagged () {
   jq -r '[{ResourceARN:.ResourceTagMappingList[] | select((.Tags? | length) < 1).ResourceARN}] | sort' <<< "${1}"
 }
 
+#Create array of objects with the topic name and parsed topic namespace
+get_topics () {
+  pushd ../services/topics > /dev/null
+  local RAW_TOPICS="$(sls invoke --stage main --function listTopics | jq -r '.[]')"
+  popd > /dev/null
+  local TOPICS=()
+  for T in $RAW_TOPICS; do
+    STAGE=$(echo "${T}" | sed 's/--/ /g' | cut -f3 -d' ')
+    TOPICS+=($(echo '{"TOPIC":"'${T}'","STAGE":"'${STAGE}'"}'))
+  done
+
+  jq -s '{TOPICS:.}' <<< ${TOPICS[*]}
+}
+
+#Produce a report with all topics and associated resource tags
+orphaned_topics () {
+  local STAGES=$(ci_inactive "${1}" | jq -r '[.[].STAGE] | sort | unique | {STAGES: [{"STAGE":.[]}]}')
+  local TOPICS=$(get_topics)
+  jq -rs '.[0] * .[1] | [[.STAGES[].STAGE] as $stages | .TOPICS[] | 
+          select( . as $topics | $stages | index($topics.STAGE) | not)] |
+          sort_by(.STAGE)' <<< $(echo ${TOPICS}${STAGES})
+}
+
 #Execute operation
 $OP "${RESOURCES}"

.github/workflows/audit-account.yml

@@ -17,7 +17,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4  
-      - name: set variable values
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+      - name: set variable values             
         run: ./.github/build-vars.sh set_values
         env:
           AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
@@ -27,6 +30,19 @@ jobs:
         with:
           role-to-assume: ${{ env.AWS_OIDC_ROLE_TO_ASSUME }}
           aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      - uses: actions/cache@v4
+        with:
+          path: |
+            **/node_modules
+            ~/.cache/Cypress
+          key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock', 'plugins/**') }}
+      - name: set path
+        run: |
+          echo "PATH=$(pwd)/node_modules/.bin/:$PATH" >> $GITHUB_ENV
+      - name: List all topics for project
+        run: |
+          #Executing the run command is required to assemble all the dependencies in the topics service
+          ./run list-topics     
       - name: Collect resources from account
         run: pushd .github && aws resourcegroupstaggingapi get-resources > resources.json
       - name: List active resources created by CI pipeline
@@ -36,7 +52,9 @@ jobs:
       - name: List resources created by Cloudformation but not from CI pipeline
         run: pushd .github && ./audit-account.sh cf_other resources.json
       - name: List untagged resources
-        run: pushd .github && ./audit-account.sh untagged resources.json            
+        run: pushd .github && ./audit-account.sh untagged resources.json
+      - name: List orphaned topics
+        run: pushd .github && ./audit-account.sh orphaned_topics                
       - name: Create reports dir
         run: pushd .github && mkdir -p reports
       - name: Assemble CSV files
@@ -56,6 +74,9 @@ jobs:
           UNTAGGED="$(./audit-account.sh untagged resources.json)"
           [[ $(jq -r 'length' <<< "${UNTAGGED}") -gt 0 ]] && jq -r '(.[0] 
             | keys_unsorted) as $keys | $keys, map([.[ $keys[] ]])[] | @csv' <<< "${UNTAGGED}" > reports/untagged.csv
+          TOPICS="$(./audit-account.sh orphaned_topics)"
+          [[ $(jq -r 'length' <<< "${TOPICS}") -gt 0 ]] && jq -r '(.[0] 
+            | keys_unsorted) as $keys | $keys, map([.[ $keys[] ]])[] | @csv' <<< "${TOPICS}" > reports/orphaned_topics.csv          
       - name: Upload reports
         uses: actions/upload-artifact@v4
         with:

.github/workflows/destroy.yml

@@ -54,4 +54,7 @@ jobs:
           role-to-assume: ${{ env.AWS_OIDC_ROLE_TO_ASSUME }}
           aws-region: ${{ env.AWS_DEFAULT_REGION }}
       - name: Destroy
-        run: ./run destroy --stage $STAGE_PREFIX$branch_name --verify false
+        # destroy app-api first due to a dependency between it and database
+        run: |
+          ./run destroy --stage $STAGE_PREFIX$branch_name --verify false --service app-api
+          ./run destroy --stage $STAGE_PREFIX$branch_name --verify false