Automatic Regression Tests in VMs #393
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Automatic Regression Tests in VMs | |
| on: | |
| push: | |
| branches: [master, ci] | |
| schedule: | |
| - cron: '22 1 * * *' | |
| workflow_dispatch: | |
| # Even with defaults, jobs not started via dispatch will only have blank inputs | |
| inputs: | |
| libass_repo: | |
| description: 'An instance of a libass git repo' | |
| required: false | |
| default: 'https://github.com/libass/libass.git' | |
| libass_ref: | |
| description: 'Git ref of libass repo to run tests on; defaults to newest commit on default branch' | |
| required: false | |
| default: '' | |
| jobs: | |
| ART-VM: | |
| # Currently only MacOS 12 runners can do virtualisation | |
| runs-on: macos-12 | |
| # Bot-friendly names | |
| name: ${{ matrix.name }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: 'ART (netbsd64, NetBSD-9, amd64)' | |
| vbox: NetBSD/NetBSD-9.1 | |
| # Leak detection appears to find false positives in libc and external libs | |
| confenv: 'ASAN_OPTIONS=detect_leaks=0 CC="gcc -fsanitize=address -fsanitize=undefined,float-cast-overflow -fno-sanitize-recover=all"' | |
| confflags: '' | |
| runenv: 'ASAN_OPTIONS=detect_leaks=0' | |
| - name: 'ART (illumos32, OpenIndiana, i686)' | |
| vbox: openindiana/hipster | |
| confenv: 'CC="gcc -m32 -msse -msse2 -mfpmath=sse"' | |
| confflags: '--host=i686-pc-solaris2.11' | |
| art_reg_skip: 'font_nonunicode' | |
| env: | |
| VAGRANT_DISABLE_VBOXSYMLINKCREATE: 1 | |
| defaults: | |
| run: | |
| shell: 'bash /tmp/artci/vm-shell {0}' | |
| steps: | |
| - name: Prepare VM-Shell | |
| shell: bash | |
| run: | | |
| mkdir -p /tmp/artci | |
| echo ' | |
| #!/bin/sh | |
| if [ "$#" -ne 1 ] ; then | |
| echo "Usage: $0 <script-file>" 1>&2 | |
| exit 1 | |
| fi | |
| set -ex | |
| vagrant upload "$1" /tmp/step.sh | |
| vagrant ssh -- "sh -e /tmp/step.sh" | |
| ' > /tmp/artci/vm-shell | |
| sudo chmod a+x /tmp/artci/vm-shell | |
| - name: Create Vagrantfile | |
| shell: bash | |
| run: | | |
| echo ' | |
| Vagrant.configure("2") do |config| | |
| config.vm.box = "'"${{ matrix.vbox }}"'" | |
| config.vm.boot_timeout = 6000 | |
| config.vm.synced_folder ".", "/vagrant", disabled: true | |
| config.ssh.insert_key = false | |
| end | |
| ' > Vagrantfile | |
| - name: Check Newest Box Version | |
| id: config | |
| shell: bash | |
| run: | | |
| pip3 install --user jq | |
| FULLBOX="${{ matrix.vbox }}" | |
| owner="${FULLBOX%%/*}" | |
| box="${FULLBOX#*/}" | |
| version="$( | |
| curl https://app.vagrantup.com/"$owner"/boxes/"$box".json \ | |
| | jq -r '[.versions[].version] | max' | |
| )" | |
| cache_key="vagrant_${FULLBOX}_${version}" | |
| echo "cache_key=${cache_key}" >> $GITHUB_OUTPUT | |
| - name: Retrieve Cached Images | |
| id: cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: package.box | |
| key: ${{ steps.config.outputs.cache_key }} | |
| - name : Import Cached Image | |
| if: steps.cache.outputs.cache-hit == 'true' | |
| shell: bash | |
| run: | | |
| vagrant box add --name "${{ matrix.vbox }}" package.box | |
| - name: Fetch Remote Image and Prepare for Cache | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| vagrant box add --provider virtualbox "${{ matrix.vbox }}" | |
| # We can only pacakge local environments not the fetched box directly | |
| # To create the environment machine must be booted once | |
| # (but for packaging it must be shutdown again) | |
| vagrant up | |
| vagrant halt | |
| vagrant package --output package.box | |
| - name: Start VM | |
| shell: bash | |
| run: | | |
| vagrant up | |
| vagrant ssh -c 'uname -a ; whoami' | |
| # The OI image is too old to be able to install dependencies without an upgrade first | |
| # and while this isn't the case with the NetBSD image (yet), the packages aren't recent either | |
| - name: Upgrade System | |
| run: | | |
| case "${{ matrix.vbox }}" in | |
| openindiana/*) | |
| pfexec pkg uninstall system/library/processor || : | |
| pfexec pkg update -r --no-backup-be || : | |
| ;; | |
| */NetBSD*) | |
| # Remove preinstalled ruby packages to speed up upgrade | |
| sudo pkgin -y remove $(pkgin list | awk '/^ruby/ {print $1}') \ | |
| || cat /var/db/pkgin/pkg_install-err.log | |
| sudo pkgin -y autoremove | |
| sudo pkgin update | |
| sudo pkgin -y full-upgrade | |
| ;; | |
| esac | |
| # On OpenIndiana upgrades of base components require a restart, since | |
| # they'll get installed into a new boot environment not the current one | |
| - name: Reboot VM | |
| if: startsWith(matrix.vbox, 'openindiana') | |
| shell: bash | |
| run: | | |
| vagrant reload | |
| - name: Install Packages | |
| run: | | |
| case "${{ matrix.vbox }}" in | |
| openindiana/*) | |
| pfexec pkg install --no-backup-be \ | |
| crypto/ca-certificates \ | |
| developer/versioning/git \ | |
| developer/gcc-10 \ | |
| developer/assembler/nasm \ | |
| developer/build/autoconf \ | |
| developer/build/automake \ | |
| developer/build/libtool \ | |
| developer/build/pkg-config \ | |
| system/library/freetype-2 \ | |
| library/fribidi \ | |
| library/c++/harfbuzz \ | |
| system/library/fontconfig \ | |
| image/library/libpng | |
| ;; | |
| */NetBSD*) | |
| # We'll use the base system's version of gcc | |
| sudo pkgin -y install \ | |
| mozilla-rootcerts-openssl git nasm \ | |
| autoconf automake libtool-base pkg-config \ | |
| freetype fribidi harfbuzz fontconfig png | |
| ;; | |
| esac | |
| - name: Checkout Git Repos | |
| run: | | |
| if [ -z "${{ github.event.inputs.libass_repo }}" ] ; then | |
| LIBASS_REPO="https://github.com/libass/libass.git" | |
| else | |
| LIBASS_REPO="${{ github.event.inputs.libass_repo }}" | |
| fi | |
| # libass | |
| echo "Cloning Libass Repo: $LIBASS_REPO" | |
| git clone --depth=1 "$LIBASS_REPO" libass | |
| cd libass | |
| if [ ! -z "${{ github.event.inputs.libass_ref }}" ] ; then | |
| echo "Checking out non-default commit..." | |
| git fetch --depth=1 origin "${{ github.event.inputs.libass_ref }}":artci_laref | |
| git checkout --force artci_laref | |
| fi | |
| echo "Testing libass commit:" | |
| git log -1 --format=%H | |
| cd .. | |
| echo "" | |
| # regression tests | |
| git clone --depth=1 'https://github.com/${{ github.repository }}.git' libass-tests | |
| cd libass-tests | |
| if [ x"${{ github.ref }}" != x ] ; then | |
| git fetch --depth=1 origin ${{ github.ref }}:artci_laref | |
| git checkout --force artci_laref | |
| else | |
| echo "Could not determine ref! Fallback to current master." | |
| fi | |
| echo "Using testsuite from commit:" | |
| git log -1 --format=%H | |
| - name: Build libass | |
| run: | | |
| cd libass | |
| ./autogen.sh | |
| ${{ matrix.confenv }} ./configure ${{ matrix.confflags }} --enable-fuzz --enable-compare | |
| make -j 2 | |
| - name: Run Regression Tests | |
| run: | | |
| cd libass-tests | |
| ${{ matrix.runenv }} \ | |
| ART_REG_SKIP="${{ matrix.art_reg_skip }}" \ | |
| ./run.sh ../libass/compare/compare ../libass/fuzz/fuzz |