From be4ce9ff65bb40f11b5e2be05aaf21f52c25e9c4 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Wed, 22 May 2024 16:37:09 -0500
Subject: [PATCH 01/12] pants: test w/ correct python and only test
 pants-plugins under py3.9

---
 .github/workflows/test.yaml | 18 ++++++++++++++----
 pylint_plugins/BUILD        |  1 +
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 7b39b61a72..b19649a405 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -88,6 +88,7 @@ jobs:
           git submodule foreach 'git tag'
 
       - name: 'Set up Python (${{ matrix.python-version }})'
+        id: python
         uses: actions/setup-python@v5
         with:
           python-version: '${{ matrix.python-version }}'
@@ -102,19 +103,28 @@ jobs:
           # To ignore a bad cache, bump the cache* integer.
           gha-cache-key: cache0-py${{ matrix.python-version }}
 
-      - name: Test
+      - name: Test pants-plugins
+        if: ${{ matrix.python-version-short == '3.9' }}
         env:
           # Github Actions uses the 'runner' user, so use that instead of stanley.
           ST2TESTS_SYSTEM_USER: 'runner'
+        run: |
+          pants test pants-plugins/::
+
+      - name: Unit Tests
         # We do not support running pytest everywhere yet. When we do it will be simply:
         #   pants test ::
         # Until then, we need to manually adjust this command line to test what we can.
-        run: |
-          pants test pylint_plugins/:: pants-plugins/::
+        run: >
+          pants test
+          --python-bootstrap-search-path=[]
+          --python-bootstrap-search-path=${{ steps.python.outputs.python-path }}
+          --tags=unit
+          pylint_plugins/::
 
       - name: Upload pants log
         uses: actions/upload-artifact@v4
         with:
           name: pants-log-py${{ matrix.python-version }}
           path: .pants.d/pants.log
-        if: always()  # We want the log even on failures.
+        if: ${{ always() }}  # We want the log even on failures.
diff --git a/pylint_plugins/BUILD b/pylint_plugins/BUILD
index 1f7bfde6c6..9705181eec 100644
--- a/pylint_plugins/BUILD
+++ b/pylint_plugins/BUILD
@@ -8,6 +8,7 @@ python_sources()
 
 python_tests(
     name="tests",
+    tags=["unit"],
     dependencies=[
         "./fixtures",
         "!//conftest.py:test_utils",

From 3661a0ba67d1f13043136e105f2cc08f292f2582 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Wed, 22 May 2024 16:43:11 -0500
Subject: [PATCH 02/12] correct order of pants args

---
 .github/workflows/test.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index b19649a405..550fcf65e7 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -116,10 +116,11 @@ jobs:
         #   pants test ::
         # Until then, we need to manually adjust this command line to test what we can.
         run: >
-          pants test
+          pants
           --python-bootstrap-search-path=[]
           --python-bootstrap-search-path=${{ steps.python.outputs.python-path }}
-          --tags=unit
+          --tag=unit
+          test
           pylint_plugins/::
 
       - name: Upload pants log

From 970aaaac447e3484f6e3476553238253d86342b8 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 12 Sep 2024 16:06:36 -0500
Subject: [PATCH 03/12] pants: run all pack and unit tests

---
 .github/workflows/test.yaml | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 550fcf65e7..807ddef161 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -111,17 +111,33 @@ jobs:
         run: |
           pants test pants-plugins/::
 
+      # We do not support running pytest everywhere yet. When we do it will be simply:
+      #   pants test ::
+      # Until then, we need to manually adjust this command line to test what we can.
+      # So far, this includes unit, pack, and pants-plugins tests.
+      # TODO: run integration tests
+
       - name: Unit Tests
-        # We do not support running pytest everywhere yet. When we do it will be simply:
-        #   pants test ::
-        # Until then, we need to manually adjust this command line to test what we can.
+        env:
+          # Github Actions uses the 'runner' user, so use that instead of stanley.
+          ST2TESTS_SYSTEM_USER: 'runner'
         run: >
           pants
           --python-bootstrap-search-path=[]
           --python-bootstrap-search-path=${{ steps.python.outputs.python-path }}
           --tag=unit
-          test
-          pylint_plugins/::
+          test '::'
+
+      - name: Pack Tests
+        env:
+          # Github Actions uses the 'runner' user, so use that instead of stanley.
+          ST2TESTS_SYSTEM_USER: 'runner'
+        run: >
+          pants
+          --python-bootstrap-search-path=[]
+          --python-bootstrap-search-path=${{ steps.python.outputs.python-path }}
+          --tag=pack
+          test '::'
 
       - name: Upload pants log
         uses: actions/upload-artifact@v4

From c401e720972d83acf69316c0aa3d829300796d98 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 24 Oct 2024 14:53:46 -0500
Subject: [PATCH 04/12] update changelog entry

---
 CHANGELOG.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 02f80e1d79..abd62f9cc8 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -69,7 +69,7 @@ Added
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
   #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253
-  #6254 #6258 #6259 #6260
+  #6254 #6258 #6259 #6260 #6269
   Contributed by @cognifloyd
 * Build of ST2 EL9 packages #6153
   Contributed by @amanda11

From 838b8e1921f51d3fdb3edf9fe67d6cd59158e073 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 24 Oct 2024 15:32:22 -0500
Subject: [PATCH 05/12] split test workflow into unit and pack test jobs

---
 .github/workflows/test.yaml | 105 +++++++++++++++++++++++++++++++-----
 1 file changed, 91 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 807ddef161..59eb048265 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -23,8 +23,8 @@ on:
   #  - cron:  '0 0 * * *'
 
 jobs:
-  test:
-    name: '${{ matrix.name }} - Python ${{ matrix.python-version-short }}'
+  unit-tests:
+    name: 'Unit Tests (pants runs: pytest) - Python ${{ matrix.python-version-short }}'
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -32,11 +32,9 @@ jobs:
         # NOTE: We need to use full Python version as part of Python deps cache key otherwise
         # setup virtualenv step will fail.
         include:
-          - name: 'Test (pants runs: pytest)'
-            python-version-short: '3.8'
+          - python-version-short: '3.8'
             python-version: '3.8.10'
-          - name: 'Test (pants runs: pytest)'
-            python-version-short: '3.9'
+          - python-version-short: '3.9'
             python-version: '3.9.14'
 
     services:
@@ -93,7 +91,6 @@ jobs:
         with:
           python-version: '${{ matrix.python-version }}'
 
-
       - name: Cache and Install APT Dependencies
         uses: ./.github/actions/apt-packages
 
@@ -111,12 +108,6 @@ jobs:
         run: |
           pants test pants-plugins/::
 
-      # We do not support running pytest everywhere yet. When we do it will be simply:
-      #   pants test ::
-      # Until then, we need to manually adjust this command line to test what we can.
-      # So far, this includes unit, pack, and pants-plugins tests.
-      # TODO: run integration tests
-
       - name: Unit Tests
         env:
           # Github Actions uses the 'runner' user, so use that instead of stanley.
@@ -128,6 +119,90 @@ jobs:
           --tag=unit
           test '::'
 
+      - name: Upload pants log
+        uses: actions/upload-artifact@v4
+        with:
+          name: pants-log-py${{ matrix.python-version }}
+          path: .pants.d/pants.log
+        if: always()  # We want the log even on failures.
+
+  pack-tests:
+    name: 'Pack Tests (pants runs: pytest) - Python ${{ matrix.python-version-short }}'
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        # NOTE: We need to use full Python version as part of Python deps cache key otherwise
+        # setup virtualenv step will fail.
+        include:
+          - python-version-short: '3.8'
+            python-version: '3.8.10'
+          - python-version-short: '3.9'
+            python-version: '3.9.14'
+
+    services:
+      mongo:
+        image: mongo:7.0
+        ports:
+          - 27017:27017
+
+      rabbitmq:
+        image: rabbitmq:3.8-management
+        options: >-
+          --name rabbitmq
+        ports:
+          - 5671:5671/tcp   # AMQP SSL port
+          - 5672:5672/tcp   # AMQP standard port
+          - 15672:15672/tcp # Management: HTTP, CLI
+
+      redis:
+         # Docker Hub image
+         image: redis
+         # Set health checks to wait until redis has started
+         options: >-
+           --name "redis"
+           --health-cmd "redis-cli ping"
+           --health-interval 10s
+           --health-timeout 5s
+           --health-retries 5
+         ports:
+           - 6379:6379/tcp
+
+    env:
+      COLUMNS: '120'
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # a test uses a submodule, and pants needs access to it to calculate deps.
+          submodules: 'recursive'
+          # sadly, the submodule will only have fetch-depth=1, which is what we want
+          # for st2.git, but not for the submodules. We still want actions/checkout
+          # to do the initial checkout, however, so that it adds auth for fetching
+          # in the submodule.
+
+      - name: Fetch repository submodules
+        run: |
+          git submodule status
+          git submodule foreach 'git fetch --all --tags'
+          git submodule foreach 'git tag'
+
+      - name: 'Set up Python (${{ matrix.python-version }})'
+        id: python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '${{ matrix.python-version }}'
+
+      - name: Cache and Install APT Dependencies
+        uses: ./.github/actions/apt-packages
+
+      - name: Initialize Pants and its GHA caches
+        uses: ./.github/actions/init-pants
+        with:
+          # To ignore a bad cache, bump the cache* integer.
+          gha-cache-key: cache0-py${{ matrix.python-version }}
+
       - name: Pack Tests
         env:
           # Github Actions uses the 'runner' user, so use that instead of stanley.
@@ -144,4 +219,6 @@ jobs:
         with:
           name: pants-log-py${{ matrix.python-version }}
           path: .pants.d/pants.log
-        if: ${{ always() }}  # We want the log even on failures.
+        if: always()  # We want the log even on failures.
+
+  #integration-tests: TODO: run integration tests

From eb3868aee098828af8c5ee0d60dde16d91886c3a Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 24 Oct 2024 15:33:23 -0500
Subject: [PATCH 06/12] Prepare to make pants workflows required

Using the Merge OK jobs simplifies setting up the branch protection rules.
---
 .github/workflows/lint.yaml | 29 +++++++++++++++++++++++++++++
 .github/workflows/test.yaml | 31 +++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)

diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
index 8d0eff683a..babf1d766a 100644
--- a/.github/workflows/lint.yaml
+++ b/.github/workflows/lint.yaml
@@ -57,3 +57,32 @@ jobs:
           name: pants-log-py${{ matrix.python-version }}
           path: .pants.d/pants.log
         if: always()  # We want the log even on failures.
+
+  set_merge_ok:
+    name: Set Merge OK (Lint)
+    if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
+    needs:
+      - lint-checks
+    outputs:
+      merge_ok: ${{ steps.set_merge_ok.outputs.merge_ok }}
+    runs-on: ubuntu-latest
+    steps:
+      - id: set_merge_ok
+        run: echo 'merge_ok=true' >> ${GITHUB_OUTPUT}
+
+  merge_ok:
+    name: Merge OK (Lint)
+    if: always()
+    needs:
+      - set_merge_ok
+    runs-on: ubuntu-latest
+    steps:
+      - run: |
+          merge_ok="${{ needs.set_merge_ok.outputs.merge_ok }}"
+          if [[ "${merge_ok}" == "true" ]]; then
+            echo "Merge OK"
+            exit 0
+          else
+            echo "Merge NOT OK"
+            exit 1
+          fi
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 59eb048265..a7d26b8edf 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -222,3 +222,34 @@ jobs:
         if: always()  # We want the log even on failures.
 
   #integration-tests: TODO: run integration tests
+
+  set_merge_ok:
+    name: Set Merge OK (Tests)
+    if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
+    needs:
+      - unit-tests
+      - pack-tests
+      #- integration-tests
+    outputs:
+      merge_ok: ${{ steps.set_merge_ok.outputs.merge_ok }}
+    runs-on: ubuntu-latest
+    steps:
+      - id: set_merge_ok
+        run: echo 'merge_ok=true' >> ${GITHUB_OUTPUT}
+
+  merge_ok:
+    name: Merge OK (Tests)
+    if: always()
+    needs:
+      - set_merge_ok
+    runs-on: ubuntu-latest
+    steps:
+      - run: |
+          merge_ok="${{ needs.set_merge_ok.outputs.merge_ok }}"
+          if [[ "${merge_ok}" == "true" ]]; then
+            echo "Merge OK"
+            exit 0
+          else
+            echo "Merge NOT OK"
+            exit 1
+          fi

From 8f16a1982442195e79dcd490180b7963d49bbbe6 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 24 Oct 2024 15:57:37 -0500
Subject: [PATCH 07/12] separate pants-plugins tests and shard unit tests

---
 .github/workflows/test.yaml | 128 +++++++++++++++++++++++++++++-------
 1 file changed, 106 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index a7d26b8edf..c83155459b 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -23,19 +23,17 @@ on:
   #  - cron:  '0 0 * * *'
 
 jobs:
-  unit-tests:
-    name: 'Unit Tests (pants runs: pytest) - Python ${{ matrix.python-version-short }}'
+  pants-plugins-tests:
+    name: 'Pants Plugins Tests (pants runs: pytest) - Python ${{ matrix.python.version-short }}'
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
         # NOTE: We need to use full Python version as part of Python deps cache key otherwise
         # setup virtualenv step will fail.
-        include:
-          - python-version-short: '3.8'
-            python-version: '3.8.10'
-          - python-version-short: '3.9'
-            python-version: '3.9.14'
+        python:
+          # Pants itself uses only 3.9
+          - {version-short: '3.9', version: '3.9.14'}
 
     services:
       mongo:
@@ -85,11 +83,11 @@ jobs:
           git submodule foreach 'git fetch --all --tags'
           git submodule foreach 'git tag'
 
-      - name: 'Set up Python (${{ matrix.python-version }})'
+      - name: 'Set up Python (${{ matrix.python.version }})'
         id: python
         uses: actions/setup-python@v5
         with:
-          python-version: '${{ matrix.python-version }}'
+          python-version: '${{ matrix.python.version }}'
 
       - name: Cache and Install APT Dependencies
         uses: ./.github/actions/apt-packages
@@ -98,16 +96,102 @@ jobs:
         uses: ./.github/actions/init-pants
         with:
           # To ignore a bad cache, bump the cache* integer.
-          gha-cache-key: cache0-py${{ matrix.python-version }}
+          gha-cache-key: cache0-py${{ matrix.python.version }}
 
       - name: Test pants-plugins
-        if: ${{ matrix.python-version-short == '3.9' }}
         env:
           # Github Actions uses the 'runner' user, so use that instead of stanley.
           ST2TESTS_SYSTEM_USER: 'runner'
         run: |
           pants test pants-plugins/::
 
+      - name: Upload pants log
+        uses: actions/upload-artifact@v4
+        with:
+          name: pants-log-py${{ matrix.python.version }}-pants-plugins-tests
+          path: .pants.d/pants.log
+        if: always()  # We want the log even on failures.
+
+  unit-tests:
+    name: 'Unit Tests Shard ${{ matrix.shard.k }}/${{ matrix.shard.n }} (pants runs: pytest) - Python ${{ matrix.python.version-short }}'
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        # NOTE: We need to use full Python version as part of Python deps cache key otherwise
+        # setup virtualenv step will fail.
+        python:
+          - {version-short: '3.8', version: '3.8.10'}
+          - {version-short: '3.9', version: '3.9.14'}
+        shard:
+          # Sharding of tests is handled by pants:
+          # https://www.pantsbuild.org/stable/docs/using-pants/advanced-target-selection#sharding-the-input-targets
+          - {k: '0', n: '2'}
+          - {k: '1', n: '2'}
+
+    services:
+      mongo:
+        image: mongo:7.0
+        ports:
+          - 27017:27017
+
+      rabbitmq:
+        image: rabbitmq:3.8-management
+        options: >-
+          --name rabbitmq
+        ports:
+          - 5671:5671/tcp   # AMQP SSL port
+          - 5672:5672/tcp   # AMQP standard port
+          - 15672:15672/tcp # Management: HTTP, CLI
+
+      redis:
+         # Docker Hub image
+         image: redis
+         # Set health checks to wait until redis has started
+         options: >-
+           --name "redis"
+           --health-cmd "redis-cli ping"
+           --health-interval 10s
+           --health-timeout 5s
+           --health-retries 5
+         ports:
+           - 6379:6379/tcp
+
+    env:
+      COLUMNS: '120'
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # a test uses a submodule, and pants needs access to it to calculate deps.
+          submodules: 'recursive'
+          # sadly, the submodule will only have fetch-depth=1, which is what we want
+          # for st2.git, but not for the submodules. We still want actions/checkout
+          # to do the initial checkout, however, so that it adds auth for fetching
+          # in the submodule.
+
+      - name: Fetch repository submodules
+        run: |
+          git submodule status
+          git submodule foreach 'git fetch --all --tags'
+          git submodule foreach 'git tag'
+
+      - name: 'Set up Python (${{ matrix.python.version }})'
+        id: python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '${{ matrix.python.version }}'
+
+      - name: Cache and Install APT Dependencies
+        uses: ./.github/actions/apt-packages
+
+      - name: Initialize Pants and its GHA caches
+        uses: ./.github/actions/init-pants
+        with:
+          # To ignore a bad cache, bump the cache* integer.
+          gha-cache-key: cache0-py${{ matrix.python.version }}
+
       - name: Unit Tests
         env:
           # Github Actions uses the 'runner' user, so use that instead of stanley.
@@ -117,28 +201,27 @@ jobs:
           --python-bootstrap-search-path=[]
           --python-bootstrap-search-path=${{ steps.python.outputs.python-path }}
           --tag=unit
+          --test-shard=${{ matrix.shard.k }}/${{ matrix.shard.n }}
           test '::'
 
       - name: Upload pants log
         uses: actions/upload-artifact@v4
         with:
-          name: pants-log-py${{ matrix.python-version }}
+          name: pants-log-py${{ matrix.python.version }}-unit-tests-shard-${{ matrix.shard.k }}_${{ matrix.shard.n }}
           path: .pants.d/pants.log
         if: always()  # We want the log even on failures.
 
   pack-tests:
-    name: 'Pack Tests (pants runs: pytest) - Python ${{ matrix.python-version-short }}'
+    name: 'Pack Tests (pants runs: pytest) - Python ${{ matrix.python.version-short }}'
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
         # NOTE: We need to use full Python version as part of Python deps cache key otherwise
         # setup virtualenv step will fail.
-        include:
-          - python-version-short: '3.8'
-            python-version: '3.8.10'
-          - python-version-short: '3.9'
-            python-version: '3.9.14'
+        python:
+          - {version-short: '3.8', version: '3.8.10'}
+          - {version-short: '3.9', version: '3.9.14'}
 
     services:
       mongo:
@@ -188,11 +271,11 @@ jobs:
           git submodule foreach 'git fetch --all --tags'
           git submodule foreach 'git tag'
 
-      - name: 'Set up Python (${{ matrix.python-version }})'
+      - name: 'Set up Python (${{ matrix.python.version }})'
         id: python
         uses: actions/setup-python@v5
         with:
-          python-version: '${{ matrix.python-version }}'
+          python-version: '${{ matrix.python.version }}'
 
       - name: Cache and Install APT Dependencies
         uses: ./.github/actions/apt-packages
@@ -201,7 +284,7 @@ jobs:
         uses: ./.github/actions/init-pants
         with:
           # To ignore a bad cache, bump the cache* integer.
-          gha-cache-key: cache0-py${{ matrix.python-version }}
+          gha-cache-key: cache0-py${{ matrix.python.version }}
 
       - name: Pack Tests
         env:
@@ -217,7 +300,7 @@ jobs:
       - name: Upload pants log
         uses: actions/upload-artifact@v4
         with:
-          name: pants-log-py${{ matrix.python-version }}
+          name: pants-log-py${{ matrix.python.version }}-pack-tests
           path: .pants.d/pants.log
         if: always()  # We want the log even on failures.
 
@@ -227,6 +310,7 @@ jobs:
     name: Set Merge OK (Tests)
     if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
     needs:
+      - pants-plugins-tests
       - unit-tests
       - pack-tests
       #- integration-tests

From 9a532ad418a8c1264fd560b5130e93e6d9a465ca Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 24 Oct 2024 16:11:36 -0500
Subject: [PATCH 08/12] CI: more unit test shards for faster feedback

---
 .github/workflows/test.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index c83155459b..b0819ecb1d 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -126,8 +126,10 @@ jobs:
         shard:
           # Sharding of tests is handled by pants:
           # https://www.pantsbuild.org/stable/docs/using-pants/advanced-target-selection#sharding-the-input-targets
-          - {k: '0', n: '2'}
-          - {k: '1', n: '2'}
+          - {k: '0', n: '4'}
+          - {k: '1', n: '4'}
+          - {k: '2', n: '4'}
+          - {k: '3', n: '4'}
 
     services:
       mongo:

From 16b48992110db717a8d1e85d60975f2d4295175b Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 24 Oct 2024 17:08:59 -0500
Subject: [PATCH 09/12] pants: tell pants that some tests use rabbitmq, redis

---
 st2actions/tests/unit/BUILD            | 13 ++++++++++---
 st2actions/tests/unit/policies/BUILD   |  2 +-
 st2api/tests/unit/controllers/v1/BUILD | 24 +++++++++++++++++++-----
 3 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/st2actions/tests/unit/BUILD b/st2actions/tests/unit/BUILD
index e8c10aa3c7..ae3f66d70a 100644
--- a/st2actions/tests/unit/BUILD
+++ b/st2actions/tests/unit/BUILD
@@ -8,10 +8,17 @@ python_tests(
     uses=["mongo"],
     overrides={
         (
-            "test_execution_cancellation.py",
-            "test_runner_container.py",
-            "test_worker.py",
+            "test_executions.py",
+            "test_policies.py",
+            "test_scheduler.py",
+            "test_workflow_engine.py",
         ): dict(
+            uses=["mongo", "rabbitmq", "redis"],
+        ),
+        ("test_execution_cancellation.py", "test_worker.py"): dict(
+            uses=["mongo", "rabbitmq", "redis", "system_user"],
+        ),
+        "test_runner_container.py": dict(
             uses=["mongo", "system_user"],
         ),
         (
diff --git a/st2actions/tests/unit/policies/BUILD b/st2actions/tests/unit/policies/BUILD
index 66a22040d1..729db50258 100644
--- a/st2actions/tests/unit/policies/BUILD
+++ b/st2actions/tests/unit/policies/BUILD
@@ -4,5 +4,5 @@ python_tests(
         "st2common.runners.runner",
         "st2common.metrics.driver",
     ],
-    uses=["mongo"],
+    uses=["mongo", "rabbitmq", "redis"],
 )
diff --git a/st2api/tests/unit/controllers/v1/BUILD b/st2api/tests/unit/controllers/v1/BUILD
index 5ef4f983f2..dd007bca19 100644
--- a/st2api/tests/unit/controllers/v1/BUILD
+++ b/st2api/tests/unit/controllers/v1/BUILD
@@ -8,12 +8,26 @@ python_tests(
     uses=["mongo"],
     overrides={
         (
-            "test_alias_execution.py",
-            "test_auth.py",
-            "test_auth_api_keys.py",
-            "test_executions.py",
-            "test_inquiries.py",
+            "test_actions.py",
+            "test_action_alias.py",
+            "test_executions_filters.py",
+            "test_kvps.py",
+            "test_packs.py",
+            "test_rules.py",
+            "test_sensortypes.py",
+            "test_triggers.py",
+            "test_triggertypes.py",
+            "test_triggerinstances.py",
         ): dict(
+            uses=["mongo", "rabbitmq", "redis"],
+        ),
+        ("test_alias_execution.py", "test_executions.py", "test_inquiries.py"): dict(
+            uses=["mongo", "rabbitmq", "redis", "system_user"],
+        ),
+        "test_service_registry.py": dict(
+            uses=["mongo", "redis"],
+        ),
+        ("test_auth.py", "test_auth_api_keys.py"): dict(
             uses=["mongo", "system_user"],
         ),
         "test_webhooks.py": dict(

From 433cad5f5ef76c59d170d34097fe9419c72cebb4 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 24 Oct 2024 21:20:20 -0500
Subject: [PATCH 10/12] Suggest ST2TESTS_REDIS_HOST/PORT in error/warning
 messages

This will hopefully help people (like me) who forget to set these vars.
---
 pants-plugins/uses_services/redis_rules.py    | 29 ++++++++++++++++---
 .../uses_services/redis_rules_test.py         |  3 +-
 pants.toml                                    |  3 ++
 st2common/st2common/services/coordination.py  | 10 +++++++
 4 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/pants-plugins/uses_services/redis_rules.py b/pants-plugins/uses_services/redis_rules.py
index 9530011b0d..5863bb1aa3 100644
--- a/pants-plugins/uses_services/redis_rules.py
+++ b/pants-plugins/uses_services/redis_rules.py
@@ -27,6 +27,7 @@
     VenvPexProcess,
     rules as pex_rules,
 )
+from pants.core.goals.test import TestExtraEnv
 from pants.engine.fs import CreateDigest, Digest, FileContent
 from pants.engine.rules import collect_rules, Get, MultiGet, rule
 from pants.engine.process import FallibleProcessResult, ProcessCacheScope
@@ -57,7 +58,12 @@ class UsesRedisRequest:
 
     #   with our version of oslo.config (newer are slower) we can't directly override opts w/ environment variables.
 
-    coord_url: str = "redis://127.0.0.1:6379"
+    host: str = "127.0.0.1"
+    port: str = "6379"
+
+    @property
+    def coord_url(self) -> str:
+        return f"redis://{self.host}:{self.port}"
 
 
 @dataclass(frozen=True)
@@ -80,9 +86,13 @@ def is_applicable(cls, target: Target) -> bool:
 )
 async def redis_is_running_for_pytest(
     request: PytestUsesRedisRequest,
+    test_extra_env: TestExtraEnv,
 ) -> PytestPluginSetup:
+    redis_host = test_extra_env.env.get("ST2TESTS_REDIS_HOST", "127.0.0.1")
+    redis_port = test_extra_env.env.get("ST2TESTS_REDIS_PORT", "6379")
+
     # this will raise an error if redis is not running
-    _ = await Get(RedisIsRunning, UsesRedisRequest())
+    _ = await Get(RedisIsRunning, UsesRedisRequest(host=redis_host, port=redis_port))
 
     return PytestPluginSetup()
 
@@ -133,6 +143,13 @@ async def redis_is_running(
     if is_running:
         return RedisIsRunning()
 
+    env_vars_hint = dedent(
+        """
+        You can also export the ST2TESTS_REDIS_HOST and ST2TESTS_REDIS_PORT
+        env vars to automatically use any redis host, local or remote,
+        while running unit and integration tests.
+        """
+    )
     # redis is not running, so raise an error with instructions.
     raise ServiceMissingError.generate(
         platform=platform,
@@ -145,16 +162,20 @@ async def redis_is_running(
                 """\
                 sudo yum -y install redis
                 # Don't forget to start redis.
+
                 """
-            ),
+            )
+            + env_vars_hint,
             service_start_cmd_deb="systemctl start redis",
             not_installed_clause_deb="this is one way to install it:",
             install_instructions_deb=dedent(
                 """\
                 sudo apt-get install -y mongodb redis
                 # Don't forget to start redis.
+
                 """
-            ),
+            )
+            + env_vars_hint,
             service_start_cmd_generic="systemctl start redis",
         ),
     )
diff --git a/pants-plugins/uses_services/redis_rules_test.py b/pants-plugins/uses_services/redis_rules_test.py
index 53e8808c37..93d7668698 100644
--- a/pants-plugins/uses_services/redis_rules_test.py
+++ b/pants-plugins/uses_services/redis_rules_test.py
@@ -73,7 +73,8 @@ def test_redis_is_running(rule_runner: RuleRunner) -> None:
 @pytest.mark.parametrize("mock_platform", platform_samples)
 def test_redis_not_running(rule_runner: RuleRunner, mock_platform: Platform) -> None:
     request = UsesRedisRequest(
-        coord_url="redis://127.100.20.7:10",  # 10 is an unassigned port, unlikely to be used
+        host="127.100.20.7",
+        port="10",  # 10 is an unassigned port, unlikely to be used
     )
 
     with pytest.raises(ExecutionError) as exception_info:
diff --git a/pants.toml b/pants.toml
index fb6272b5b0..72e5fcb6c8 100644
--- a/pants.toml
+++ b/pants.toml
@@ -240,6 +240,9 @@ extra_env_vars = [
   # Use this so that the test system does not require the stanley user.
   # For example: export ST2TESTS_SYSTEM_USER=${USER}
   "ST2TESTS_SYSTEM_USER",
+  # Use these to override the redis host and port
+  "ST2TESTS_REDIS_HOST",
+  "ST2TESTS_REDIS_PORT",
 ]
 
 [twine]
diff --git a/st2common/st2common/services/coordination.py b/st2common/st2common/services/coordination.py
index 8f693f56e5..803b8f694d 100644
--- a/st2common/st2common/services/coordination.py
+++ b/st2common/st2common/services/coordination.py
@@ -15,6 +15,8 @@
 
 from __future__ import absolute_import
 
+import sys
+
 import six
 
 from oslo_config import cfg
@@ -244,9 +246,17 @@ def get_coordinator(start_heart=True, use_cache=True):
     global COORDINATOR
 
     if not configured():
+        extra_msg = ""
+        # sys._called_from_test set in conftest.py for pytest runs
+        if "nose" in sys.modules.keys() or hasattr(sys, "_called_from_test"):
+            extra_msg = (
+                " Set ST2TESTS_REDIS_HOST and ST2TESTS_REDIS_PORT env vars to "
+                "configure the coordination backend for unit and integration tests."
+            )
         LOG.warning(
             "Coordination backend is not configured. Code paths which use coordination "
             "service will use best effort approach and race conditions are possible."
+            f"{extra_msg}"
         )
 
     if not use_cache:

From df96ef0de56669c7af5513c8556e37f54360449e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 24 Oct 2024 21:21:52 -0500
Subject: [PATCH 11/12] add ST2TESTS_REDIS_HOST/PORT vars to pants CI workflow

---
 .github/workflows/test.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index b0819ecb1d..9be21ee106 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -102,6 +102,8 @@ jobs:
         env:
           # Github Actions uses the 'runner' user, so use that instead of stanley.
           ST2TESTS_SYSTEM_USER: 'runner'
+          ST2TESTS_REDIS_HOST: '127.0.0.1'
+          ST2TESTS_REDIS_PORT: '6379'
         run: |
           pants test pants-plugins/::
 
@@ -198,6 +200,8 @@ jobs:
         env:
           # Github Actions uses the 'runner' user, so use that instead of stanley.
           ST2TESTS_SYSTEM_USER: 'runner'
+          ST2TESTS_REDIS_HOST: '127.0.0.1'
+          ST2TESTS_REDIS_PORT: '6379'
         run: >
           pants
           --python-bootstrap-search-path=[]
@@ -292,6 +296,8 @@ jobs:
         env:
           # Github Actions uses the 'runner' user, so use that instead of stanley.
           ST2TESTS_SYSTEM_USER: 'runner'
+          ST2TESTS_REDIS_HOST: '127.0.0.1'
+          ST2TESTS_REDIS_PORT: '6379'
         run: >
           pants
           --python-bootstrap-search-path=[]

From 37125e713bd3e8d966810e07fad58fd389aef7e5 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 24 Oct 2024 22:44:27 -0500
Subject: [PATCH 12/12] Work around missing indexes error in a test

---
 st2api/tests/unit/controllers/v1/test_triggertypes.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/st2api/tests/unit/controllers/v1/test_triggertypes.py b/st2api/tests/unit/controllers/v1/test_triggertypes.py
index 2461796e20..99d224c709 100644
--- a/st2api/tests/unit/controllers/v1/test_triggertypes.py
+++ b/st2api/tests/unit/controllers/v1/test_triggertypes.py
@@ -21,6 +21,7 @@
 import six
 
 from st2api.controllers.v1.triggers import TriggerTypeController
+from st2common.models.db.trigger import TriggerTypeDB
 
 from st2tests.api import FunctionalTest
 from st2tests.api import APIControllerWithIncludeAndExcludeFilterTestCase
@@ -57,6 +58,9 @@ class TriggerTypeControllerTestCase(
     include_attribute_field_name = "payload_schema"
     exclude_attribute_field_name = "parameters_schema"
 
+    ensure_indexes = True
+    ensure_indexes_models = [TriggerTypeDB]
+
     @classmethod
     def setUpClass(cls):
         # super's setUpClass does the following: