Pants ci: Run self-check tests w/ venv generated by pants (#6287)

.github/workflows/test.yaml

@@ -519,13 +519,13 @@ jobs:
           test '::'
 
       - name: Compress Service Logs Before upload
-        if: ${{ failure() }}
+        if: failure()
         run: |
           ./tools/launchdev.sh stop # stop st2 before collecting logs
           tar cvzpf logs.tar.gz logs/*
 
       - name: Upload StackStorm services Logs
-        if: ${{ failure() }}
+        if: failure()
         uses: actions/upload-artifact@v4
         with:
           name: logs-py${{ matrix.python-version }}-st2cluster-integration-tests
@@ -539,6 +539,160 @@ jobs:
           path: .pants.d/pants.log
         if: always()  # We want the log even on failures.
 
+  self-check:
+    name: 'Self-Check - Python ${{ matrix.python.version-short }}'
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        # NOTE: We need to use full Python version as part of Python deps cache key otherwise
+        # setup virtualenv step will fail.
+        python:
+          - {version-short: '3.8', version: '3.8.10'}
+          - {version-short: '3.9', version: '3.9.14'}
+
+    services:
+      mongo:
+        image: mongo:7.0
+        ports:
+          - 27017:27017
+
+      rabbitmq:
+        image: rabbitmq:3.8-management
+        options: >-
+          --name rabbitmq
+        ports:
+          - 5671:5671/tcp   # AMQP SSL port
+          - 5672:5672/tcp   # AMQP standard port
+          - 15672:15672/tcp # Management: HTTP, CLI
+
+      redis:
+         # Docker Hub image
+         image: redis
+         # Set health checks to wait until redis has started
+         options: >-
+           --name "redis"
+           --health-cmd "redis-cli ping"
+           --health-interval 10s
+           --health-timeout 5s
+           --health-retries 5
+         ports:
+           - 6379:6379/tcp
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # a test uses a submodule, and pants needs access to it to calculate deps.
+          submodules: 'recursive'
+          # sadly, the submodule will only have fetch-depth=1, which is what we want
+          # for st2.git, but not for the submodules. We still want actions/checkout
+          # to do the initial checkout, however, so that it adds auth for fetching
+          # in the submodule.
+
+      - name: Fetch repository submodules
+        run: |
+          git submodule status
+          git submodule foreach 'git fetch --all --tags'
+          git submodule foreach 'git tag'
+
+      - name: 'Set up Python (${{ matrix.python.version }})'
+        id: python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '${{ matrix.python.version }}'
+
+      - name: Cache and Install APT Dependencies
+        uses: ./.github/actions/apt-packages
+
+      - name: Initialize Pants and its GHA caches
+        uses: ./.github/actions/init-pants
+        with:
+          # To ignore a bad cache, bump the cache* integer.
+          gha-cache-key: cache0-py${{ matrix.python.version }}
+
+      - name: Export virtualenv to run Dev ST2 Cluster
+        run: >
+          pants
+          --python-bootstrap-search-path=[]
+          --python-bootstrap-search-path=${{ steps.python.outputs.python-path }}
+          export
+          --resolve=st2
+
+      - name: Add ST2 System User Key
+        env:
+          ST2_CI_USER: ${{ env.ST2TESTS_SYSTEM_USER }}
+          ST2_SYSTEM_USER__USER: ${{ env.ST2TESTS_SYSTEM_USER }}
+          ST2_SYSTEM_USER__SSH_KEY_FILE: /home/${{ env.ST2TESTS_SYSTEM_USER }}/.ssh/stanley_rsa
+        run: |
+          sudo -E ./scripts/ci/add-itest-user-key.sh
+
+      - name: Launch Dev ST2 Cluster
+        env:
+          # NOTE: ST2_CONF defaults to ${GITHUB_WORKSPACE}/conf/st2.dev.conf
+          VIRTUALENV_DIR: ./dist/export/python/virtualenvs/st2/${{ steps.python.outputs.python-version }}
+          ST2_SYSTEM_USER__USER: ${{ env.ST2TESTS_SYSTEM_USER }}
+          ST2_SYSTEM_USER__SSH_KEY_FILE: /home/${{ env.ST2TESTS_SYSTEM_USER }}/.ssh/stanley_rsa
+        run: |
+          sudo -E ./scripts/github/prepare-integration.sh
+
+      - name: Extend the path for upcoming tasks
+        # pants uses PEP 660 editable wheels to add our code to the virtualenv.
+        # But PEP 660 editable wheels do not include 'scripts'.
+        # https://peps.python.org/pep-0660/#limitations
+        # So, we need to include each bin dir in PATH instead of virtualenv/bin.
+        env:
+          VIRTUALENV_DIR: dist/export/python/virtualenvs/st2/${{ steps.python.outputs.python-version }}
+        run: |
+          for component_bin in ${GITHUB_WORKSPACE}/st2*/bin; do
+          echo ${component_bin} | tee -a $GITHUB_PATH
+          done
+          echo ${GITHUB_WORKSPACE}/${VIRTUALENV_DIR}/bin | tee -a $GITHUB_PATH
+
+      - name: Create symlinks to find the binaries when running st2 actions
+        # st2 is actually a console_script entry point, not just a 'script'
+        # so it IS included in the virtualenv. But, st2-run-pack-tests might not be included.
+        env:
+          VIRTUALENV_DIR: dist/export/python/virtualenvs/st2/${{ steps.python.outputs.python-version }}
+        run: |
+          ln -s ${GITHUB_WORKSPACE}/${VIRTUALENV_DIR}/bin/st2 /usr/local/bin/st2
+          ln -s ${GITHUB_WORKSPACE}/st2common/bin/st2-run-pack-tests /usr/local/bin/st2-run-pack-tests
+
+      - name: Run st2-self-check
+        env:
+          # Space separated list of tests to be skipped if the self-check is running in GitHub Actions
+          TESTS_TO_SKIP: "tests.test_quickstart_rules tests.test_run_pack_tests_tool"
+          ST2_SYSTEM_USER__USER: ${{ env.ST2TESTS_SYSTEM_USER }}
+          ST2_SYSTEM_USER__SSH_KEY_FILE: /home/${{ env.ST2TESTS_SYSTEM_USER }}/.ssh/stanley_rsa
+        run: >
+          sudo
+          -E
+          ST2_AUTH_TOKEN=$(st2 auth testu -p 'testp' -t)
+          ST2_CONF=${GITHUB_WORKSPACE}/conf/st2.dev.conf
+          PATH=${PATH}
+          st2common/bin/st2-self-check
+
+      - name: Compress Service Logs Before upload
+        if: failure()
+        run: |
+          ./tools/launchdev.sh stop # stop st2 before collecting logs
+          tar cvzpf logs.tar.gz logs/*
+
+      - name: Upload StackStorm services Logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-py${{ matrix.python-version }}-self-check
+          path: logs.tar.gz
+          retention-days: 7
+
+      - name: Upload pants log
+        uses: actions/upload-artifact@v4
+        with:
+          name: pants-log-py${{ matrix.python.version }}-self-check
+          path: .pants.d/pants.log
+        if: always()  # We want the log even on failures.
+
   set_merge_ok:
     name: Set Merge OK (Tests)
     if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
@@ -548,6 +702,7 @@ jobs:
       - pack-tests
       - integration-tests
       - integration-st2cluster-tests
+      - self-check
     outputs:
       merge_ok: ${{ steps.set_merge_ok.outputs.merge_ok }}
     runs-on: ubuntu-latest

CHANGELOG.rst

@@ -69,7 +69,7 @@ Added
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
   #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253
-  #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273
+  #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287
   Contributed by @cognifloyd
 * Build of ST2 EL9 packages #6153
   Contributed by @amanda11

scripts/ci/add-itest-user-key.sh

@@ -20,24 +20,26 @@ chmod 0700 ~/.ssh
 
 chown -R "${ST2_CI_USER}:${ST2_CI_USER}" ~/.ssh
 
+SYSTEM_USER=${ST2TESTS_SYSTEM_USER:-${ST2_SYSTEM_USER__USER:-stanley}}
+
 # Create an SSH system user (default `stanley` user may be already created)
-if (! id stanley 2>/dev/null); then
-  useradd stanley
+if (! id "${SYSTEM_USER}" 2>/dev/null); then
+  useradd "${SYSTEM_USER}"
 fi
 
-mkdir -p /home/stanley/.ssh
+mkdir -p "/home/${SYSTEM_USER}/.ssh"
 
 # Generate ssh keys on StackStorm box and copy over public key into remote box.
-ssh-keygen -f /home/stanley/.ssh/stanley_rsa -P ""
+ssh-keygen -f "/home/${SYSTEM_USER}/.ssh/stanley_rsa" -P ""
 
 # Authorize key-base acces
-sh -c 'cat /home/stanley/.ssh/stanley_rsa.pub >> /home/stanley/.ssh/authorized_keys'
-chmod 0600 /home/stanley/.ssh/authorized_keys
-chmod 0700 /home/stanley/.ssh
-chown -R stanley:stanley /home/stanley
+sh -c "cat /home/${SYSTEM_USER}/.ssh/stanley_rsa.pub >> /home/${SYSTEM_USER}/.ssh/authorized_keys"
+chmod 0600 "/home/${SYSTEM_USER}/.ssh/authorized_keys"
+chmod 0700 "/home/${SYSTEM_USER}/.ssh"
+chown -R "${SYSTEM_USER}:${SYSTEM_USER}" "/home/${SYSTEM_USER}"
 
 # Apply sudo fix for GHA runner user
 sh -c 'echo "runner ALL=(ALL)       NOPASSWD: SETENV: ALL" >> /etc/sudoers.d/st2'
 # Enable passwordless sudo for 'stanley' user
-sh -c 'echo "stanley    ALL=(ALL)       NOPASSWD: SETENV: ALL" >> /etc/sudoers.d/st2'
+sh -c "echo '${SYSTEM_USER}    ALL=(ALL)       NOPASSWD: SETENV: ALL' >> /etc/sudoers.d/st2"
 chmod 0440 /etc/sudoers.d/st2

st2common/bin/st2-self-check

@@ -113,15 +113,18 @@ fi
 cp -R --preserve st2tests/packs/* /opt/stackstorm/packs/
 
 echo "Installing asserts, fixtures, tests and examples packs."
-st2 run packs.setup_virtualenv packs=examples,tests,asserts,fixtures,webui
-st2ctl reload --register-all
+st2 run packs.setup_virtualenv packs=examples,tests,asserts,fixtures,webui || ((ERRORS++))
+st2ctl reload --register-all || ((ERRORS++))
 
 echo "Deleting 'st2tests' directory from `pwd`."
 rm -R st2tests/
 popd
 
-# Retrieve test action list
-TEST_ACTION_LIST=`st2 action list --pack=tests -w 90 | awk '{ print $2 }' | grep -v "|" | grep -v "ref" | grep tests.test_`
+TEST_ACTION_LIST=""
+if [ $ERRORS -eq 0 ]; then
+  # Retrieve test action list
+  TEST_ACTION_LIST=`st2 action list --pack=tests -w 90 | awk '{ print $2 }' | grep -v "|" | grep -v "ref" | grep tests.test_`
+fi
 
 # Run all the tests
 for TEST in $TEST_ACTION_LIST

st2common/bin/st2ctl

@@ -141,7 +141,7 @@ function service_manager() {
 function reopen_component_log_files() {
   COM=${1}
 
-  PID=`ps ax | grep -v grep | grep -v st2ctl | grep -E "(${COM}\.wsgi)|(bin/${COM})|(hubot .*${COM})" | awk '{print $1}'`
+  PID=`ps axww | grep -v grep | grep -v st2ctl | grep -E "(${COM}\.wsgi)|(bin/${COM})|(hubot .*${COM})" | awk '{print $1}'`
   if [[ ! -z ${PID} ]]; then
     for p in ${PID}; do
       echo "Sending SIGUSR1 to ${COM} PID: ${p}"
@@ -201,7 +201,7 @@ function getpids() {
   COMPONENTS=${COMPONENTS}
 
   for COM in ${COMPONENTS}; do
-    PID=`ps ax | grep -v grep | grep -v st2ctl | grep -E "(${COM}\.wsgi)|(bin/${COM})|(hubot .*${COM})" | awk '{print $1}'`
+    PID=`ps axww | grep -v grep | grep -v st2ctl | grep -E "(${COM}\.wsgi)|(bin/${COM})|(hubot .*${COM})" | awk '{print $1}'`
 
     if [[ ! -z ${PID} ]]; then
       for p in ${PID}; do

st2reactor/tests/integration/test_sensor_watcher.py

@@ -31,6 +31,15 @@ class SensorWatcherTestCase(IntegrationTestCase):
     def setUpClass(cls):
         super(SensorWatcherTestCase, cls).setUpClass()
 
+    def setUp(self):
+        super().setUp()
+        # pre-condition: Make sure there is no test pollution
+        sw_queues = self._get_sensor_watcher_amqp_queues(
+            queue_name="st2.sensor.watch.covfefe"
+        )
+        # TODO: Maybe just delete any leftover queues from previous failed test runs.
+        self.assertTrue(len(sw_queues) == 0)
+
     def test_sensor_watch_queue_gets_deleted_on_stop(self):
         def create_handler(sensor_db):
             pass
@@ -64,7 +73,8 @@ def delete_handler(sensor_db):
         )
         self.assertTrue(len(sw_queues) == 0)
 
-    def _list_amqp_queues(self):
+    @staticmethod
+    def _list_amqp_queues():
         rabbit_client = Client("localhost:15672", "guest", "guest")
         queues = [q["name"] for q in rabbit_client.get_queues()]
         return queues

tools/launchdev.sh

@@ -140,7 +140,7 @@ function init()
         export ST2_SYSTEM_USER__USER="${ST2TESTS_SYSTEM_USER}"
         ST2VARS+=("ST2_SYSTEM_USER__USER")
     fi
-    if [ -n "${ST2TESTS_REDIS_HOST}" ] && [ -n "${ST2TESTS_REDIS_PORT}"]; then
+    if [ -n "${ST2TESTS_REDIS_HOST}" ] && [ -n "${ST2TESTS_REDIS_PORT}" ]; then
         export ST2_COORDINATION__URL="redis://${ST2TESTS_REDIS_HOST}:${ST2TESTS_REDIS_PORT}?namespace=_st2_dev"
         ST2VARS+=("ST2_COORDINATION__URL")
     fi