Merge pull request #432 from robotology/diegoferigo-patch-1 #916
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD | |
| on: | |
| push: | |
| branches: ["**"] | |
| tags-ignore: ["**"] | |
| pull_request: | |
| workflow_dispatch: | |
| release: | |
| types: [published] | |
| jobs: | |
| # ============= | |
| build-colcon: | |
| # ============= | |
| name: 'colcon@${{ matrix.ignition }}' | |
| if: | | |
| (github.event_name == 'push' && github.ref != 'refs/heads/master') || | |
| (github.event_name == 'pull_request' && github.event.pull_request.base.ref != 'master') | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ignition: | |
| # - dome | |
| # - edifice | |
| - fortress | |
| env: | |
| CCACHE_DIR: ${{ github.workspace }}/.ccache | |
| steps: | |
| - name: '🔍️ Inspect Environment' | |
| run: | | |
| env | grep ^GITHUB | |
| echo "" | |
| cat ${GITHUB_EVENT_PATH} | |
| echo "" | |
| env | |
| - name: '⬇️️ Install dependencies' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| lz4 \ | |
| git \ | |
| wget \ | |
| gpg-agent \ | |
| ninja-build \ | |
| build-essential \ | |
| software-properties-common | |
| - name: '🐍 Initialize Python' | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.8 | |
| - name: '🚚 Compilation cache' | |
| uses: actions/cache@v2 | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| # We include the commit sha in the cache key, as new cache entries are | |
| # only created if there is no existing entry for the key yet. | |
| key: ${{ runner.os }}-ccache-${{ matrix.ignition }}-${{ github.sha }} | |
| # Restore any ccache cache entry, if none for the key above exists | |
| restore-keys: ${{ runner.os }}-ccache-${{ matrix.ignition }} | |
| - name: '🚚 Enable ccache' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y ccache | |
| echo "/usr/lib/ccache" >> $GITHUB_PATH | |
| ccache --set-config=max_size=5.0G | |
| ccache --set-config=sloppiness=file_macro,locale,time_macros | |
| ccache -p | |
| ccache -s | |
| - name: '⚙️ Add osrf ppa' | |
| run: | | |
| sudo sh -c 'echo "deb http://packages.osrfoundation.org/gazebo/ubuntu-stable `lsb_release -cs` main" >\ | |
| /etc/apt/sources.list.d/gazebo-stable.list' | |
| wget http://packages.osrfoundation.org/gazebo.key -O - | sudo apt-key add - | |
| sudo apt-get update | |
| - name: '⚙️ Prepare colcon workspace' | |
| run: | | |
| pip install vcstool colcon-common-extensions | |
| sudo mkdir -p /workspace/src /workspace/install | |
| sudo chmod -R a+rw /workspace | |
| cd /workspace/src | |
| wget -O - ${TAGS_YAML} | vcs import | |
| echo $(sort -u $(find . -iname 'packages-'$(lsb_release -cs)'.apt' -o -iname 'packages.apt') | grep -v -E "^libignition|^libsdformat" | tr '\n' ' ') \ | |
| > /workspace/install/pkg.txt | |
| xargs -a /workspace/install/pkg.txt sudo apt-get install -y --no-install-recommends | |
| env: | |
| TAGS_YAML: https://raw.githubusercontent.com/ignition-tooling/gazebodistro/master/collection-${{ matrix.ignition }}.yaml | |
| - name: '🏗️ Build colcon workspace' | |
| run: | | |
| cd /workspace | |
| colcon graph | |
| colcon build \ | |
| --cmake-args \ | |
| -GNinja \ | |
| -DBUILD_TESTING:BOOL=OFF \ | |
| -DCMAKE_BUILD_TYPE=Debug \ | |
| --merge-install | |
| - name: '📈 Ccache stats' | |
| run: ccache --show-stats | |
| - name: '📦️ Compress the workspace' | |
| run: tar -I lz4 -cf /tmp/workspace_install.tar.lz4 /workspace/install | |
| - name: '⬆️ Upload the workspace' | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| path: /tmp/workspace_install.tar.lz4 | |
| name: workspace-${{ matrix.ignition }} | |
| retention-days: 1 | |
| # =============== | |
| build-and-test: | |
| # =============== | |
| name: 'Build and Test [${{matrix.type}}|${{matrix.ignition}}|${{matrix.python}}]' | |
| if: always() | |
| needs: [ build-colcon ] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| type: | |
| - User | |
| - Developer | |
| ignition: | |
| # - dome | |
| # - edifice | |
| - fortress | |
| python: | |
| - 3.8 | |
| - 3.9 | |
| steps: | |
| - name: '🔍 Inspect Environment' | |
| run: | | |
| env | grep ^GITHUB | |
| echo "" | |
| cat ${GITHUB_EVENT_PATH} | |
| echo "" | |
| env | |
| - name: '⬇️ Install build dependencies' | |
| if: contains(matrix.os, 'ubuntu') | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| lz4 \ | |
| git \ | |
| wget \ | |
| cmake \ | |
| gpg-agent \ | |
| ninja-build \ | |
| build-essential \ | |
| software-properties-common | |
| - name: '🐍 Initialize Python' | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: '🔀 Clone repository' | |
| uses: actions/checkout@master | |
| - name: '🔀 Download all refs' | |
| run: git fetch --prune --unshallow | |
| # ================ | |
| # Install Ignition | |
| # ================ | |
| - name: '⚙️ Add osrf ppa' | |
| if: contains(matrix.os, 'ubuntu') | |
| run: | | |
| sudo sh -c 'echo "deb http://packages.osrfoundation.org/gazebo/ubuntu-stable `lsb_release -cs` main" >\ | |
| /etc/apt/sources.list.d/gazebo-stable.list' | |
| wget http://packages.osrfoundation.org/gazebo.key -O - | sudo apt-key add - | |
| sudo apt-get update | |
| - name: '[🔒️|stable] Install Ignition from ppa' | |
| if: | | |
| contains(matrix.os, 'ubuntu') && ( | |
| github.event_name == 'release' || | |
| github.ref == 'refs/heads/master' || | |
| (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'master') | |
| ) | |
| run: sudo apt-get install -y --no-install-recommends ignition-${{ matrix.ignition }} | |
| - name: '[🧪|nightly] Download pre-built colcon workspace' | |
| uses: actions/download-artifact@v2 | |
| if: | | |
| contains(matrix.os, 'ubuntu') && ( | |
| (github.event_name == 'push' && github.ref != 'refs/heads/master') || | |
| (github.event_name == 'pull_request' && github.event.pull_request.base.ref != 'master') | |
| ) | |
| with: | |
| path: /tmp | |
| name: workspace-${{ matrix.ignition }} | |
| - name: '[🧪|nightly] Setup colcon workspace' | |
| if: | | |
| contains(matrix.os, 'ubuntu') && ( | |
| (github.event_name == 'push' && github.ref != 'refs/heads/master') || | |
| (github.event_name == 'pull_request' && github.event.pull_request.base.ref != 'master') | |
| ) | |
| run: | | |
| sudo tar -I lz4 -xf /tmp/workspace_install.tar.lz4 -C / | |
| xargs -a /workspace/install/pkg.txt sudo apt-get install -y --no-install-recommends | |
| echo "source /workspace/install/setup.bash" | sudo tee -a /etc/bash.bashrc | |
| # ============= | |
| # Build project | |
| # ============= | |
| # This is required because ScenarIO needs to import the iDynTree targets | |
| - name: '⬇️ Install iDynTree dependencies' | |
| if: contains(matrix.os, 'ubuntu') | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| libxml2-dev coinor-libipopt-dev libeigen3-dev libassimp-dev swig | |
| - name: '🤖 Install iDynTree' | |
| run: | | |
| pip install idyntree | |
| IDYNTREE_PYTHON_PKG=$(python3 -c 'import idyntree, pathlib; print(pathlib.Path(idyntree.__file__).parent)') | |
| echo "CMAKE_PREFIX_PATH=$IDYNTREE_PYTHON_PKG" >> $GITHUB_ENV | |
| # Note: In order to execute the setup.sh script, the file /etc/bash.bashrc must be sourced. | |
| # To do that, we change the shell to a bash interactive session with 'bash -i -e'. | |
| # Developer installation | |
| - name: '[👷|developer] Build and Install C++ ScenarIO' | |
| if: matrix.type == 'Developer' | |
| shell: bash -i -e {0} | |
| run: | | |
| env | |
| cmake -S . -B build/ \ | |
| -GNinja \ | |
| -DCMAKE_BUILD_TYPE=Debug \ | |
| -DIGNITION_DISTRIBUTION=$(python3 -c "print('${{ matrix.ignition }}'.capitalize())") | |
| sudo cmake --build build/ --target install | |
| - name: '[👷|developer] Install Python ScenarIO' | |
| if: matrix.type == 'Developer' | |
| run: pip install -e ./scenario | |
| # User installation | |
| - name: '[👤|user] Create wheel (default ignition)' | |
| if: matrix.type == 'User' && matrix.ignition == 'fortress' | |
| shell: bash -i -e {0} | |
| run: pip wheel --use-feature=in-tree-build -v -w dist/ ./scenario | |
| # Note: Calling "pip wheel" with "--global-option" forces all dependencies to be built from their sdist. | |
| # Since it's very slow, we create the wheel from setup.py without isolation (requires system deps). | |
| - name: '[👤|user] Create wheel (custom ignition)' | |
| if: matrix.type == 'User' && matrix.ignition != 'fortress' | |
| shell: bash -i -e {0} | |
| run: | | |
| pip install wheel setuptools-scm cmake-build-extension | |
| python3 ./scenario/setup.py bdist_wheel \ | |
| build_ext -DIGNITION_DISTRIBUTION=$(python3 -c "print('${{ matrix.ignition }}'.capitalize())") | |
| - name: '[👤|user] Install local wheel' | |
| if: matrix.type == 'User' | |
| run: pip install -v dist/scenario-*.whl | |
| - name: '🔍️ Inspect installed ScenarIO package' | |
| if: matrix.type == 'User' && contains(matrix.os, 'ubuntu') | |
| run: | | |
| sudo apt-get install -y --no-install-recommends tree | |
| tree $(python3 -c "import scenario, pathlib; print(pathlib.Path(scenario.__file__).parent)") | |
| # ==================== | |
| # Install gym-ignition | |
| # ==================== | |
| - name: '🐍 Install gym-ignition' | |
| run: pip install wheel && pip install .[all] | |
| # ============ | |
| # Test project | |
| # ============ | |
| - name: '🔍 Inspect installed versions' | |
| run: pip list | grep -E "^scenario|^gym-ignition" | |
| - name: '[🐍|scenario] Python Tests' | |
| shell: bash -i -e {0} | |
| run: pytest -m "scenario" | |
| - name: '[🚨|scenario] Python Tests with Valgrind' | |
| shell: bash -i -e {0} | |
| if: failure() | |
| run: | | |
| sudo apt-get install -y --no-install-recommends valgrind | |
| pip install colour-valgrind | |
| valgrind --log-file=/tmp/valgrind.log pytest -m "scenario" || colour-valgrind -t /tmp/valgrind.log | |
| - name: '[🐍|gym-ignition] Python Tests' | |
| shell: bash -i -e {0} | |
| run: pytest -m "gym_ignition" | |
| - name: '[🚨|gym-ignition] Python Tests with Valgrind' | |
| shell: bash -i -e {0} | |
| if: failure() | |
| run: | | |
| sudo apt-get install -y --no-install-recommends valgrind | |
| pip install colour-valgrind | |
| valgrind --log-file=/tmp/valgrind.log pytest -m "gym_ignition" || colour-valgrind -t /tmp/valgrind.log | |
| # ============================ | |
| # Upload artifacts (only User) | |
| # ============================ | |
| - name: '🗑️ Remove external wheels' | |
| if: matrix.type == 'User' | |
| run: find dist/ -type f -not -name 'scenario-*' -delete -print | |
| # We have to trick PyPI that our wheels are manylinux2014 even if they are not. | |
| # Unfortunately we cannot create self-contained wheels (neither the PEP600 perennial) | |
| # due to the Ignition architecture. | |
| - name: '📝 Rename scenario wheel' | |
| if: matrix.type == 'User' && contains(matrix.os, 'ubuntu') | |
| run: | | |
| ls dist/ | |
| find dist/ -type f -name "*.whl" -exec rename.ul linux manylinux2014 {} + | |
| ls dist/ | |
| - name: '🔍 Inspect dist folder' | |
| if: matrix.type == 'User' | |
| run: ls -lah dist/ | |
| - name: '⬆️ Upload artifacts' | |
| uses: actions/upload-artifact@v2 | |
| if: matrix.type == 'User' && matrix.ignition == 'fortress' | |
| with: | |
| path: dist/* | |
| name: dist | |
| # ======= | |
| # Website | |
| # ======= | |
| - name: '⬇️ Install website dependencies' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y doxygen texlive-font-utils | |
| - name: '🔍 Inspect metadata' | |
| run: sphinx-multiversion --dump-metadata docs/sphinx/ build/ | |
| # This is necessary because otherwise the check for uncommitted apidoc | |
| # only detects additions, not removals. | |
| - name: '🗑️ Remove apidoc folder' | |
| run: rm -r docs/sphinx/apidoc | |
| - name: '🏗️ Build sphinx website' | |
| shell: bash -i -e {0} | |
| run: | | |
| [[ -d build ]] && sudo chown -R $(id -u):$(id -g) build/ | |
| cmake -S . -B build/ -DBUILD_DOCS:BOOL=ON | |
| cmake --build build/ --target sphinx | |
| - name: '🔍 Check new uncommitted apidoc' | |
| run: test $(git status --porcelain | wc -l) -eq 0 | |
| - name: '🔍 git status' | |
| if: ${{ failure() }} | |
| run: | | |
| git status | |
| git diff | |
| - name: '⬆️ Upload website folder' | |
| uses: actions/upload-artifact@v2 | |
| if: matrix.type == 'User' && matrix.ignition == 'fortress' && contains(matrix.os, 'ubuntu') | |
| with: | |
| path: build/html | |
| name: website | |
| retention-days: 1 | |
| # =================== | |
| website-deployment: | |
| # =================== | |
| name: 'Website Deployment' | |
| if: always() && needs.build-and-test.result == 'success' | |
| needs: [ build-and-test ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: '⬇️ Download website folder' | |
| uses: actions/download-artifact@v2 | |
| with: | |
| path: build/html | |
| name: website | |
| - name: '🔍 Inspect html folder' | |
| run: ls -lah build/html | |
| - name: '🚀 Deploy' | |
| uses: JamesIves/github-pages-deploy-action@releases/v3 | |
| if: | | |
| github.event_name == 'push' && | |
| github.repository == 'robotology/gym-ignition' && | |
| (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/devel') | |
| with: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BRANCH: gh-pages | |
| FOLDER: build/html | |
| CLEAN: true | |
| CLEAN_EXCLUDE: '[".gitignore", ".nojekyll"]' | |
| # ================= | |
| python-packaging: | |
| # ================= | |
| name: 'Python packaging' | |
| if: always() && needs.build-and-test.result == 'success' | |
| needs: [ build-and-test ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: '🔍️ Inspect Environment' | |
| run: | | |
| env | grep ^GITHUB | |
| echo "" | |
| cat ${GITHUB_EVENT_PATH} | |
| echo "" | |
| env | |
| # Any Python3 version is ok in this job | |
| - name: '🐍 Initialize Python' | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.8 | |
| - name: '🔀 Clone repository' | |
| uses: actions/checkout@master | |
| - name: '🔀 Download all refs' | |
| run: git fetch --prune --unshallow | |
| # =============== | |
| # Download wheels | |
| # =============== | |
| - name: '⬇️ Download scenario wheels' | |
| uses: actions/download-artifact@v2 | |
| with: | |
| path: dist | |
| name: dist | |
| - name: '🔍 Inspect dist folder' | |
| run: ls -lah dist/ | |
| # =============== | |
| # Create packages | |
| # =============== | |
| # We use build to create sdist. Also pipx would work. | |
| # https://github.com/pypa/build | |
| # Compared to calling setup.py, the advantage of these tools is that | |
| # they automatically handle the build dependencies. | |
| - name: '🐍️️ Install dependencies' | |
| run: pip install build | |
| - name: '[📦|scenario]️ Create sdist' | |
| run: python -m build --sdist scenario/ -o dist/ | |
| - name: '[📦|gym-ignition]️ Create sdist and wheel' | |
| run: python -m build . | |
| # ================ | |
| # Upload artifacts | |
| # ================ | |
| - name: '🗑️ Remove external packages' | |
| run: find dist/ -type f -not \( -name '*scenario-*' -o -name '*gym_ignition-*' \) -delete -print | |
| - name: '🔍 Check packages' | |
| run: pipx run twine check dist/* | |
| - name: '🔍 Inspect dist folder' | |
| run: ls -lah dist/ | |
| - name: '⬆️ Upload artifacts' | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| path: dist/* | |
| name: dist | |
| # ============ | |
| upload_pypi: | |
| # ============ | |
| name: Publish to PyPI | |
| if: always() && needs.build-and-test.result == 'success' && needs.python-packaging.result == 'success' | |
| needs: | |
| - build-and-test | |
| - python-packaging | |
| runs-on: ubuntu-latest | |
| # Devel branch produces pre-releases. | |
| # GitHub releases produce stable releases. | |
| steps: | |
| # Needed only to extract from the git repo the last revision | |
| - name: '🔀 Clone repository' | |
| uses: actions/checkout@master | |
| - name: '🔀 Download all refs' | |
| run: git fetch --prune --unshallow | |
| - name: '⬇️ Download Python packages' | |
| uses: actions/download-artifact@v2 | |
| with: | |
| path: dist | |
| name: dist | |
| - name: '🔍 Inspect dist folder' | |
| run: ls -lah dist/ | |
| # Validate the last tag accordingly to PEP440 | |
| # From https://stackoverflow.com/a/37972030/12150968 | |
| - name: '📌 Check PEP440 compliance' | |
| if: github.event_name == 'release' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y source-highlight | |
| last_tag_with_v="$(git describe --abbrev=0 --tags)" | |
| last_tag=${last_tag_with_v#v} | |
| rel_regexp='^(\d+!)?(\d+)(\.\d+)+([\.\-\_])?((a(lpha)?|b(eta)?|c|r(c|ev)?|pre(view)?)\d*)?(\.?(post|dev)\d*)?$' | |
| echo "" | |
| echo $last_tag | |
| echo "" | |
| check-regexp ${rel_regexp} ${last_tag} | |
| match=$(check-regexp ${rel_regexp} ${last_tag} | grep matches | cut -d ' ' -f 5) | |
| test $match -eq 1 && true | |
| - name: '⬆️ Publish packages to PyPI' | |
| if: | | |
| github.repository == 'robotology/gym-ignition' && | |
| ((github.event_name == 'release' && github.event.action == 'published') || | |
| (github.event_name == 'push' && github.ref == 'refs/heads/devel')) | |
| uses: pypa/gh-action-pypi-publish@master | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_TOKEN }} | |
| skip_existing: true |