Merge pull request #314 from nextstrain/trs/ci/use-singularity-3.x #940
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| tags: | |
| # Version tags. | |
| # | |
| # Tags matching this pattern will cause the "release" job below to run, | |
| # so edit it carefully! It should not match arbitrary tags. | |
| - "[0-9]+.[0-9]+.[0-9]+*" | |
| pull_request: | |
| workflow_dispatch: | |
| # Routinely check that we continue to work in the face of external changes. | |
| schedule: | |
| # Every day at 17:42 UTC / 9:42 Seattle (winter) / 10:42 Seattle (summer) | |
| - cron: "42 17 * * *" | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: nextstrain/.github/actions/shellcheck@master | |
| test-source: | |
| name: test-source (python=${{ matrix.python }} os=${{ matrix.os }}) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - macos-latest | |
| - windows-latest | |
| python: | |
| - '3.6' | |
| - '3.7' | |
| - '3.8' | |
| - '3.9' | |
| - '3.10' | |
| # Python 3.6 is not available on Ubuntu 22.04, which is what | |
| # ubuntu-latest points to as of Q4 2022¹, so replace that job with 3.6 | |
| # on Ubuntu 20.04. | |
| # -trs, 22 Nov 2022 | |
| # | |
| # ¹ <https://github.blog/changelog/2022-11-09-github-actions-ubuntu-latest-workflows-will-use-ubuntu-22-04/> | |
| # | |
| # A bodged https://github.com/actions/python-version build of 3.7.17 | |
| # for macOS mistakenly doesn't include the bz2 module of the stdlib¹, | |
| # which breaks us because some of our deps (fsspec, at the least) | |
| # assume bz2 is available² (though that will be fixed in its next | |
| # release³). Replace the 3.7 macOS job with a 3.7.16 job. | |
| # -trs, 22 June 2023 | |
| # | |
| # ¹ <https://github.com/actions/setup-python/issues/682> | |
| # ² <https://github.com/nextstrain/cli/pull/289#issuecomment-1595417903> | |
| # ³ <https://github.com/fsspec/filesystem_spec/pull/1295> | |
| exclude: | |
| - os: ubuntu-latest | |
| python: '3.6' | |
| - os: macos-latest | |
| python: '3.7' | |
| include: | |
| - os: ubuntu-20.04 | |
| python: '3.6' | |
| - os: macos-latest | |
| python: '3.7.16' | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Install Nextstrain CLI | |
| run: python3 -m pip install --upgrade '.[dev]' | |
| - run: ./devel/pytest -v | |
| build-dist: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.10" | |
| # Install up-to-date packaging toolchain. | |
| - run: python3 -m pip install --upgrade pip setuptools wheel | |
| - run: python3 -m pip install --upgrade build | |
| # Update version to include local git rev if we're not building a release tag. | |
| - if: github.ref_type != 'tag' | |
| run: | | |
| version="$(./devel/read-version)" | |
| git_rev="$(git rev-parse --short @)" | |
| ./devel/update-version "${version%%+*}+git.${git_rev}" | |
| # Build dists. | |
| - run: python3 -m build | |
| # Upload dists as workflow artifacts. | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: dist | |
| path: dist/ | |
| build-standalone: | |
| needs: build-dist | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Generally we want to build on the oldest supported OS to maximize the | |
| # final binary's compatibility. See pyoxidizer's docs for more | |
| # considerations affecting the choice of build machine OS¹ and future | |
| # plans for robust, turnkey build environments². | |
| # | |
| # On Linux, we build inside a container to avoid portability issues, so | |
| # the container's host OS version here doesn't have an impact (other | |
| # than CI stability). | |
| # | |
| # ¹ https://pyoxidizer.readthedocs.io/en/stable/pyoxidizer_distributing_binary_portability.html | |
| # ² https://pyoxidizer.readthedocs.io/en/stable/pyoxidizer_status.html#an-official-build-environment | |
| # | |
| # XXX TODO: GitHub doesn't yet host any runners on M1 (Apple Silicon, | |
| # aarch64, arm64) hardware, and we don't have M1 hardware on which to | |
| # self-host a runner. Amazon's mac2.metal EC2 instances are M1, but | |
| # they cost a minimum of ~$15/day. It would be cheaper to buy an M1 | |
| # Mac Mini for ~$700 which would pay for itself in less than 2 months. | |
| # -trs, 31 May 2022 | |
| include: | |
| - os: ubuntu-22.04 | |
| target: x86_64-unknown-linux-gnu | |
| exe: nextstrain | |
| - os: macos-11 | |
| target: x86_64-apple-darwin | |
| exe: nextstrain | |
| - os: windows-2019 | |
| target: x86_64-pc-windows-msvc | |
| exe: nextstrain.exe | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| # Note that this Python version doesn't impact the actual build. | |
| - uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.10" | |
| # Build the executable + necessary external files from the dists. | |
| - uses: actions/checkout@v3 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: dist | |
| path: dist/ | |
| - name: Set DIST, DIST_VERSION, and INSTALLATION_ARCHIVE_STEM | |
| run: | | |
| # shellcheck disable=SC2034 | |
| DISTS=(dist/nextstrain_cli-*-py3-none-any.whl) | |
| DIST="${DISTS[0]}" | |
| DIST_VERSION="$DIST" | |
| DIST_VERSION="${DIST_VERSION#dist/nextstrain_cli-}" | |
| DIST_VERSION="${DIST_VERSION%-py3-none-any.whl}" | |
| INSTALLATION_ARCHIVE_STEM="nextstrain-cli-${DIST_VERSION}-standalone-${{ matrix.target }}" | |
| for var in DIST DIST_VERSION INSTALLATION_ARCHIVE_STEM; do | |
| echo "${var}=${!var}" | tee -a "$GITHUB_ENV" | |
| done | |
| - run: | | |
| ./devel/pyoxidizer build \ | |
| --release \ | |
| --target-triple ${{ matrix.target }} \ | |
| --var NEXTSTRAIN_CLI_DIST "$DIST" | |
| # Analyze the executable for potential portability issues. | |
| # | |
| # This is for informational purposes only in build logs, so we don't care | |
| # if it fails. Currently it only works on Linux, though it's supposed to | |
| # eventually work on all platforms supported by pyoxidizer. | |
| - if: runner.os == 'Linux' | |
| run: ./devel/pyoxidizer analyze build/${{ matrix.target }}/release/installation/${{ matrix.exe }} | |
| continue-on-error: true | |
| # XXX TODO: Review and report on licensing of all the stuff built into | |
| # the binary, as bundling things statically can trigger different license | |
| # terms than "normal" installs (e.g. via pip). See also pyoxidizer's | |
| # docs about this and the tooling it includes to support license review.¹ | |
| # -trs, 1 June 2022 | |
| # | |
| # ¹ https://pyoxidizer.readthedocs.io/en/stable/pyoxidizer_packaging_licensing.html#licensing-considerations | |
| # Create installation archive. | |
| # | |
| # Use tar on Unix to preserve file modes (e.g. the executable bit), thus | |
| # avoiding having to restore them manually after archive extraction. Use | |
| # zip on Windows because it's a native format which requires no extra | |
| # tooling. | |
| - if: runner.os != 'Windows' | |
| run: tar czvpf "$INSTALLATION_ARCHIVE_STEM.tar.gz" -C build/${{ matrix.target }}/release/installation/ . | |
| - if: runner.os == 'Windows' | |
| run: Compress-Archive -DestinationPath "$Env:INSTALLATION_ARCHIVE_STEM.zip" -Path build/${{ matrix.target }}/release/installation/* | |
| shell: pwsh | |
| # Upload installation archive as a workflow artifact. | |
| # | |
| # At least one path needs to match, or this errors. | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: standalone-${{ matrix.target }} | |
| path: | | |
| ${{ env.INSTALLATION_ARCHIVE_STEM }}.tar.gz | |
| ${{ env.INSTALLATION_ARCHIVE_STEM }}.zip | |
| if-no-files-found: error | |
| # Quick smoke test that the executable at least runs! Useful before | |
| # launching the more extensive tests below. | |
| - run: ./build/${{ matrix.target }}/release/installation/${{ matrix.exe }} --help | |
| test-dist: | |
| needs: build-dist | |
| name: test-dist (python=${{ matrix.python }} os=${{ matrix.os }}) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - macos-latest | |
| - windows-latest | |
| python: | |
| - '3.6' | |
| - '3.7' | |
| - '3.8' | |
| - '3.9' | |
| # XXX TODO: Add 3.10 here once supported by Conda/Bioconda/Conda Forge. | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| # Add -l for setup-integration-tests → setup-miniconda → automatic | |
| # activation of "test" environment. | |
| shell: bash -l -eo pipefail {0} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| path: src/ | |
| - uses: ./src/.github/actions/setup-integration-tests | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: dist | |
| path: dist/ | |
| - name: Install Nextstrain CLI | |
| run: python3 -m pip install --upgrade dist/nextstrain_cli-*-py3-none-any.whl | |
| - uses: ./src/.github/actions/run-integration-tests | |
| test-standalone: | |
| needs: build-standalone | |
| name: test-standalone (os=${{ matrix.os}}, target=${{ matrix.target }}) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Test on all the platforms available via GitHub Actions. | |
| # | |
| # Ideally we'd test on machines with ~fresh OS installs. The kitchen | |
| # sink of development/build software pre-installed into GitHub Action's | |
| # virtual-environments has a decent risk of making this CI blind to | |
| # end-user runtime issues with our binaries (e.g. missing DLLs). Such | |
| # fresh CI machines are not readily available, however, since | |
| # pre-installation is convenient for builds. | |
| # | |
| # XXX TODO: macOS aarch64 (M1, Apple Silicon, arm64); see above. | |
| include: | |
| - { os: ubuntu-20.04, target: x86_64-unknown-linux-gnu } | |
| - { os: ubuntu-22.04, target: x86_64-unknown-linux-gnu } | |
| - { os: macos-11, target: x86_64-apple-darwin } | |
| - { os: macos-12, target: x86_64-apple-darwin } | |
| - { os: windows-2019, target: x86_64-pc-windows-msvc } | |
| - { os: windows-2022, target: x86_64-pc-windows-msvc } | |
| runs-on: ${{matrix.os}} | |
| defaults: | |
| run: | |
| # Add -l for setup-integration-tests → setup-miniconda → automatic | |
| # activation of "test" environment. | |
| shell: bash -l -eo pipefail {0} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| path: src/ | |
| - uses: ./src/.github/actions/setup-integration-tests | |
| with: | |
| python-version: '3.9' | |
| # Download and extract the installation archive. | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: standalone-${{ matrix.target }} | |
| - if: runner.os != 'Windows' | |
| run: tar xzvpf nextstrain-cli-*-standalone-${{ matrix.target }}.tar.gz | |
| - if: runner.os == 'Windows' | |
| run: Expand-Archive -Path nextstrain-cli-*-standalone-${{ matrix.target }}.zip -DestinationPath . | |
| shell: pwsh | |
| - run: echo "$PWD" >> "$GITHUB_PATH" | |
| - uses: ./src/.github/actions/run-integration-tests | |
| doc: | |
| uses: nextstrain/.github/.github/workflows/docs-ci.yaml@master | |
| with: | |
| docs-directory: doc/ | |
| pip-install-target: .[dev] | |
| make-target: dirhtml | |
| release: | |
| # Restricted to version tags by the "on: push: tags: …" config at the top. | |
| if: |2 | |
| github.event_name == 'push' | |
| && github.ref_type == 'tag' | |
| needs: | |
| - build-dist | |
| - build-standalone | |
| - test-source | |
| - test-dist | |
| - test-standalone | |
| - lint | |
| - doc | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| # In actions/checkout@v3 above, annotated tags are intentionally | |
| # **overwritten** and converted to a lightweight tag. Forcibly restore | |
| # the annotated tag object from the remote so we can verify/use it later. | |
| - run: git fetch --force origin tag "$GITHUB_REF_NAME" | |
| - uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.10" | |
| - run: python3 -m pip install --upgrade twine | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: dist | |
| path: dist/ | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: standalone-x86_64-unknown-linux-gnu | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: standalone-x86_64-apple-darwin | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: standalone-x86_64-pc-windows-msvc | |
| - run: twine upload dist/* | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} | |
| TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/ | |
| - run: ./devel/create-github-release "${{github.ref_name}}" dist/* nextstrain-cli-*-standalone-* | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |