YDA-5829 troubleshooting tool for published data packages #761
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run integration and API tests | |
| on: | |
| push: | |
| branches: | |
| - development | |
| - release-1.9 | |
| - "**-atr" | |
| # We can force an integration/API test run without opening a PR by pushing to a branch name that ends with "-atr" | |
| pull_request: | |
| jobs: | |
| build: | |
| runs-on: ubuntu-22.04 | |
| if: | | |
| ${{ ! contains(github.event.head_commit.message, '#noapitests') && | |
| ( github.event_name == 'push' || | |
| github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name | |
| ) | |
| }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.8 | |
| - name: Extract branch name | |
| shell: bash | |
| run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT | |
| id: extract_branch | |
| # For release branches that have a Docker setup, we use the Docker setup version of that release. | |
| # For other branches, we use the Docker setup of the development branch. | |
| - name: Determine Yoda repository branch | |
| run: | | |
| if [ "${{ steps.extract_branch.outputs.branch }}" = "release-1.9" ]; then | |
| echo "branch=release-1.9" >> $GITHUB_OUTPUT | |
| else | |
| echo "branch=development" >> $GITHUB_OUTPUT | |
| fi | |
| id: yoda_repo_branch | |
| - name: Install API test dependencies | |
| run: | | |
| pip3 install --user -r tests/requirements.txt | |
| sudo apt install -y docker-compose | |
| - name: Clone Yoda repo for Docker Setup | |
| run: | | |
| git clone -b "${{ steps.yoda_repo_branch.outputs.branch }}" --single-branch https://github.com/UtrechtUniversity/yoda.git | |
| - name: Prepare hosts file for API tests | |
| run: | | |
| sudo echo "127.0.0.1 portal.yoda eus.yoda data.yoda public.yoda" | sudo tee -a /etc/hosts | |
| - name: Start Dockerized Yoda | |
| run: | | |
| cd yoda/docker/compose | |
| docker-compose pull | |
| ../up.sh -d | |
| - name: Wait until Dockerized setup is ready | |
| shell: bash | |
| run: | | |
| until $(curl -k --output /dev/null --silent --head --fail https://portal.yoda:8443 ); do printf '.' ; sleep 1; done | |
| docker exec provider.yoda sh -c 'while ! pgrep irodsServer > /dev/null ; do echo Waiting for iRODS to start ... ; sleep 1; done' | |
| - name: Pull and install latest version of ruleset | |
| shell: bash | |
| run: | | |
| cd yoda/docker/compose | |
| docker exec provider.yoda sh -c 'set -x ; cd /etc/irods/yoda-ruleset && sudo chown irods:irods -R /etc/irods/yoda-ruleset && sudo -u irods git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && sudo -u irods git pull && sudo -u irods git status' | |
| docker exec provider.yoda sh -c "set -x ; cd /etc/irods/yoda-ruleset && sudo -u irods git checkout ${{ steps.extract_branch.outputs.branch }} && sudo -u irods python -m pip --no-cache-dir install --user -r /etc/irods/yoda-ruleset/requirements.txt && sudo -u irods make && sudo -u irods make install" | |
| docker exec provider.yoda sh -c "set -x ; sudo -u irods /var/lib/irods/irodsctl restart" | |
| - name: Pull and install latest version of portal | |
| shell: bash | |
| run: | | |
| cd yoda/docker/compose | |
| docker exec portal.yoda sh -c 'set -x ; cd /var/www/yoda && git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git pull' | |
| docker exec portal.yoda sh -c 'set -x ; cd /var/www/yoda && git checkout ${{ steps.extract_branch.outputs.branch }} || git checkout development' | |
| docker exec portal.yoda sh -c 'set -x ; cd /var/www/yoda && git status' | |
| docker exec portal.yoda sh -c 'set -x ; touch /var/www/yoda/*.wsgi' | |
| - name: Run integration tests | |
| shell: bash | |
| run: | | |
| cd yoda/docker/compose | |
| docker exec provider.yoda sh -c 'set -x ; cd /etc/irods/yoda-ruleset && sudo -u irods /usr/bin/irule -r irods_rule_engine_plugin-python-instance -F tools/run-integration-tests.r' > test-results.txt | |
| echo "Integration test results: " | |
| echo | |
| cat test-results.txt | |
| echo | |
| echo "Statistics:" | |
| echo | |
| echo "$(grep -c VERDICT_OK test-results.txt) test(s) OK" | |
| echo "$(grep -c VERDICT_FAILED test-results.txt) test(s) failed with unexpected return value" | |
| echo "$(grep -c VERDICT_EXCEPTION test-results.txt) test(s) failed with exception" | |
| echo "$(grep -c VERDICT_ test-results.txt) test(s) in total" | |
| echo | |
| echo "Overall result: " | |
| if grep -E "VERDICT_EXCEPTION|VERDICT_FAILED" test-results.txt> /dev/null ; then printf "Test failures detected"; exit 1; fi | |
| if grep -E "VERDICT_" test-results.txt > /dev/null ; then printf "Tests succeeded"; else printf "Failure to run tests"; exit 1; fi | |
| - name: Run API tests | |
| shell: bash | |
| run: | | |
| cd tests | |
| nohup bash -c 'while true ; do sleep 5 ; ../yoda/docker/run-cronjob.sh copytovault >> ../copytovault.log 2>&1 ; ../yoda/docker/run-cronjob.sh publication >> ../publication.log 2>&1 ; done' & | |
| test -d mycache || mkdir -p mycache | |
| python3 -m pytest --skip-ui --intake --datarequest --deposit -o cache_dir=mycache --environment environments/docker.json | |
| cat ../copytovault.log | |
| cat ../publication.log | |
| - name: Output rodsLogs | |
| if: failure() | |
| run: | | |
| docker exec provider.yoda sh -c 'set -x ; cat /var/lib/irods/log/rodsLog*' | |
| # Uncomment section below when needed for debugging. | |
| # | |
| # - name: Setup tmate session for debugging | |
| # uses: mxschmitt/action-tmate@v3 | |
| # if: ${{ failure() }} | |
| # with: | |
| # limit-access-to-actor: true |