diff --git a/.github/workflows/qe-tests.yml b/.github/workflows/qe-tests.yml
index 58cfda11768..e50531f347c 100644
--- a/.github/workflows/qe-tests.yml
+++ b/.github/workflows/qe-tests.yml
@@ -106,6 +106,15 @@ jobs:
         - name: Set up PKI deployment by running playbook on "controller"
           run: vagrant provision controller
 
+        - name: Run CA related tests
+          run: |
+                vagrant ssh controller -c \
+                'pytest --ansible-host-pattern master \
+                --ansible-inventory /vagrant/inventory \
+                --ansible-module-path /vagrant/tests/dogtag/pytest-ansible/common-modules/ \
+                /vagrant/tests/dogtag/pytest-ansible/pytest/sanity/test_role_users.py \
+                --junit-xml pki_cert_junit.xml -qsvvv'
+
         - name: Setup tmate session
           if: failure()
           uses: mxschmitt/action-tmate@v2
diff --git a/ci/Vagrantfile b/ci/Vagrantfile
index 4db062fe4f9..3756380a24b 100644
--- a/ci/Vagrantfile
+++ b/ci/Vagrantfile
@@ -22,9 +22,9 @@ Vagrant.configure("2") do |config|
       ansible.verbose = "vv"
       ansible.raw_arguments  = "-M /vagrant/tests/dogtag/pytest-ansible/common-modules/"
     end
+
     controller.vm.provision "shell", inline: "export ANSIBLE_LIBRARY=/vagrant/tests/dogtag/pytest-ansible/common-modules/"
     controller.vm.provision "shell", inline: "pip install -e /vagrant/tests/dogtag/pytest-ansible/"
-    controller.vm.provision "shell", inline: "py.test --ansible-inventory /vagrant/inventory --ansible-host-pattern master /vagrant/tests/dogtag/pytest-ansible/pytest/sanity/test_role_users.py -vvvvvvvvv"
     controller.vm.network :private_network, ip: "CONTROLLER_IP"
   end
 
@@ -41,4 +41,3 @@ Vagrant.configure("2") do |config|
 #  clone0.vm.network :private_network, ip: "CLONE_IP"
 #  end
 end
-