Automatic Regression Tests #1606
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Automatic Regression Tests | |
| on: | |
| push: | |
| branches: [master, ci] | |
| schedule: | |
| - cron: '22 2 * * *' | |
| workflow_dispatch: | |
| # Even with defaults, jobs not started via dispatch will only have blank inputs | |
| inputs: | |
| libass_repo: | |
| description: 'An instance of a libass git repo' | |
| required: false | |
| default: 'https://github.com/libass/libass.git' | |
| libass_ref: | |
| description: 'Git ref of libass repo to run tests on; defaults to newest commit on default branch' | |
| required: false | |
| default: '' | |
| jobs: | |
| ART: | |
| runs-on: ubuntu-20.04 | |
| # Run on each arch we support ASM on and furthemore | |
| # some additional little and big endian archs | |
| # ASM: amd64, x32, i386 | |
| # Big: s390x | |
| # Little: arm64, riscv64, mips64el | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: [amd64, s390x, arm64, mips64el] | |
| suite: [buster] | |
| chroot-rev: [9] | |
| confopts: [""] | |
| cflags: [""] | |
| include: | |
| # i386 needs some SSE usage enabled to get a bit-identical result (even without ASM). | |
| # This is due to different (higher) precision of 387 floating math (default for <=i686). | |
| # SSE2 was introduced in 2000 by Intel and is present in both 32bit and 64bit | |
| # CPUs from both AMD and Intel since 2003. Our ASM needs at least SSE2 anyway. | |
| - arch: i386 | |
| suite: buster | |
| cflags: '-msse -msse2 -mfpmath=sse' | |
| sanity: sane? | |
| chroot-rev: 10 | |
| # Yes, GHA's kernel supports x32 syscalls | |
| - arch: x32 | |
| suite: sid | |
| port: yes | |
| chroot-rev: 10 | |
| - arch: riscv64 | |
| suite: sid | |
| port: yes | |
| chroot-rev: 10 | |
| # Enable sanitisers for native build | |
| - arch: amd64 | |
| sanity: sane? | |
| steps: | |
| - name: Prepare System | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| debian-keyring debian-archive-keyring debian-ports-archive-keyring \ | |
| debootstrap qemu-user-static zstd \ | |
| git ca-certificates | |
| sudo mkdir -p /var/chroot/imgs | |
| sudo chmod -R 777 /var/chroot | |
| luser="$(whoami)" | |
| sudo chown $luser:$luser /var/chroot/imgs | |
| - name: Determine Configuration | |
| id: config | |
| run: | | |
| # Map debian arch/abi name to qemu arch name | |
| case "${{ matrix.arch }}" in | |
| amd64|x32) qarch="x86_64" ;; | |
| arm64) qarch="aarch64" ;; | |
| armel|armhf) qarch="arm" ;; # Untested | |
| arm) qarch="armeb" ;; # Untested | |
| ppc64el) qarch="ppc64le" ;; | |
| *) qarch="${{ matrix.arch }}" ;; | |
| esac | |
| # Make sure we have an qemu layer available | |
| if [ x"$qarch" != xi386 ] && [ x"$qarch" != xx86_64 ] \ | |
| && [ ! -x "/usr/bin/qemu-${qarch}-static" ] | |
| then | |
| echo "Arch '${{ matrix.arch }}/${qarch}' not supported!" | |
| exit 1 | |
| fi | |
| # Set prefixes as needed | |
| outer_prefix="" | |
| inner_prefix="" | |
| case "${{ matrix.arch }}" in | |
| amd64|x32) | |
| : ;; | |
| i386) | |
| outer_prefix="linux32" | |
| ;; | |
| *) | |
| inner_prefix="/usr/bin/qemu-${qarch}-static" | |
| ;; | |
| esac | |
| # Regenerate chroots regularly | |
| # Often a few GHA jobs suffered connection timeouts on regeneration, | |
| # but work just fine on manual rerun. Offset regeneration time | |
| # for some chroots in hopes to avoid this. | |
| # Note: date +%j can have leading zeros causing $(()) to interpret it as octal. | |
| # To avoid this, initially use bc to stick to decimal and remove leading zeros. | |
| if [ "x${{ matrix.port }}" = xyes ] ; then | |
| tmp="$( echo "$(date +%j) + 1" | bc )" | |
| else | |
| tmp="$(echo "$(date +%j)" | bc )" | |
| fi | |
| cache_period="$(printf "%d-%02d" "$(date +%Y)" "$(( tmp / 44 ))")" | |
| echo "cache_period=${cache_period}" >> $GITHUB_OUTPUT | |
| echo "QEMU_ARCH=${qarch}" >> $GITHUB_ENV | |
| echo "OPRE=${outer_prefix}" >> $GITHUB_ENV | |
| echo "IPRE=${inner_prefix}" >> $GITHUB_ENV | |
| echo "CHR_DIR=debian-${{ matrix.suite }}-${{ matrix.arch }}" >> $GITHUB_ENV | |
| # Each repo is allowed up to 5GB total cache | |
| # use it to store our (compressed) chroot dirs | |
| # (cache is branch scoped) | |
| - name: Retrieve Cached Chroots | |
| uses: actions/cache@v3 | |
| id: cache | |
| with: | |
| path: /var/chroot/imgs | |
| key: ${{ matrix.suite }}-${{ matrix.arch }}-${{ matrix.chroot-rev }}_${{ steps.config.outputs.cache_period }} | |
| - name: Load and Update Cached Chroot | |
| if: steps.cache.outputs.cache-hit == 'true' | |
| run: | | |
| sudo tar --zstd -xf /var/chroot/imgs/"$CHR_DIR".tar.zstd | |
| if [ ! -z "${IPRE}" ] ; then | |
| echo "Update qemu-binary '${IPRE}' ..." | |
| sudo cp "${IPRE}" "${CHR_DIR}${IPRE}" | |
| fi | |
| sudo $OPRE chroot "$CHR_DIR" $IPRE /bin/dash -c ' | |
| export DEBIAN_FRONTEND=noninteractive | |
| export LC_ALL=C.UTF-8 | |
| apt-get update && apt-get -y upgrade --with-new-pkgs | |
| cp -a /home/artci/workarea-skel /home/artci/workarea | |
| ' | |
| - name: (Re)Create Chroot | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: | | |
| echo "Creating '$CHR_DIR' !" | |
| sudo mkdir "$CHR_DIR" | |
| if [ x"${{ matrix.port }}" = xyes ] ; then | |
| crepo="http://ftp.ports.debian.org/debian-ports/" | |
| else | |
| crepo="http://deb.debian.org/debian" | |
| fi | |
| # For simplicity, pretend every arch is foreign | |
| sudo debootstrap --foreign --arch=${{ matrix.arch }} \ | |
| --variant=minbase \ | |
| --include=debian-ports-archive-keyring \ | |
| --no-check-gpg \ | |
| ${{ matrix.suite }} "$CHR_DIR" "$crepo" | |
| if [ ! -z "${IPRE}" ] ; then | |
| echo "Copy qemu-binary '${IPRE}' into chroot." | |
| sudo cp "${IPRE}" "${CHR_DIR}${IPRE}" | |
| fi | |
| # Set additional packages for some archs | |
| case "${{ matrix.arch }}" in | |
| amd64|i386|x32) add_pkgs="nasm" ;; | |
| *) add_pkgs="" ;; | |
| esac | |
| if [ -n "${{ matrix.sanity }}" ] ; then | |
| add_pkgs="$add_pkgs libasan5 libubsan1" | |
| fi | |
| # ime we don't need to mount everything for those setup steps | |
| sudo $OPRE chroot "$CHR_DIR" $IPRE /bin/dash /debootstrap/debootstrap --second-stage | |
| sudo $OPRE chroot "$CHR_DIR" $IPRE /bin/dash -c ' | |
| export DEBIAN_FRONTEND=noninteractive | |
| export LC_ALL=C.UTF-8 | |
| set -e | |
| apt-get install -y --no-install-recommends --no-install-suggests \ | |
| autoconf automake make libtool pkg-config \ | |
| gcc \ | |
| libfreetype6-dev libfribidi-dev libfontconfig-dev libharfbuzz-dev libpng-dev \ | |
| '"$add_pkgs"' | |
| groupadd -g 1000 artci | |
| useradd -m -d /home/artci -s /bin/dash -g artci -u 1000 artci | |
| runuser -u artci mkdir /home/artci/workarea-skel | |
| ' | |
| - name: Save Chroot Base to Cache | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: | | |
| # Compress and store chroot in cache dir | |
| luser="$(whoami)" | |
| sudo tar --zstd -cf /var/chroot/imgs/"$CHR_DIR".tar.zstd "$CHR_DIR" | |
| sudo chown $luser:$luser /var/chroot/imgs/"$CHR_DIR".tar.zstd | |
| # Initialise workarea for further use | |
| sudo cp -a "$CHR_DIR/home/artci/workarea-skel" "$CHR_DIR/home/artci/workarea" | |
| - name: Checkout Git Repos | |
| run: | | |
| if [ -z "${{ github.event.inputs.libass_repo }}" ] ; then | |
| LIBASS_REPO="https://github.com/libass/libass.git" | |
| else | |
| LIBASS_REPO="${{ github.event.inputs.libass_repo }}" | |
| fi | |
| sudo sh -c ' | |
| set -e | |
| cd "'"$CHR_DIR"'"/home/artci/workarea | |
| # libass | |
| echo "Cloning Libass Repo: '"$LIBASS_REPO"'" | |
| git clone --depth=1 "'"$LIBASS_REPO"'" libass | |
| cd libass | |
| if [ ! -z "'"${{ github.event.inputs.libass_ref }}"'" ] ; then | |
| echo "Checking out non-default commit..." | |
| git fetch --depth=1 origin '"${{ github.event.inputs.libass_ref }}"':artci_laref | |
| git checkout --force artci_laref | |
| fi | |
| echo "Testing libass commit:" | |
| git log -1 --format=%H | |
| cd .. | |
| echo "" | |
| # regression tests | |
| git clone --depth=1 https://github.com/${{ github.repository }}.git libass-tests | |
| cd libass-tests | |
| if [ x"${{ github.ref }}" != x ] ; then | |
| git fetch --depth=1 origin ${{ github.ref }}:artci_laref | |
| git checkout --force artci_laref | |
| else | |
| echo "Could not determine ref! Fallback to current master." | |
| fi | |
| echo "Using testsuite from commit:" | |
| git log -1 --format=%H | |
| cd .. | |
| echo "" | |
| # Fix ownership | |
| sudo chown -R 1000:1000 * | |
| ' | |
| - name: Prepare Chroot Jobs | |
| run: | | |
| cd "$CHR_DIR"/home/artci/workarea | |
| # Make sure we can write to job scripts | |
| sudo touch env.sh build-libass.sh test-libass.sh | |
| sudo chmod 777 env.sh build-libass.sh test-libass.sh | |
| echo '#!/bin/sh | |
| set -e | |
| export DEBIAN_FRONTEND=noninteractive | |
| export LC_ALL=C.UTF-8 | |
| export SANITY="'"${{ matrix.sanity }}"'" | |
| export LIBASS_CONF="'"${{ matrix.confopts }}"'" | |
| export LIBASS_CFLAGS="'"${{ matrix.cflags }}"'" | |
| printf "Toolchain: %s\\n" "$(gcc -dumpmachine)" | |
| printf "Kernel-Arch: %s\\n" "$(uname -m)" | |
| printf "Debian Ver.: %s\\n" "$(cat /etc/debian_version)" | |
| printf "Sanity-Check: %s\\n" "$SANITY" | |
| echo "" | |
| ' > env.sh | |
| # Build libass | |
| echo '#!/bin/sh | |
| . ./env.sh | |
| cd libass | |
| ./autogen.sh | |
| CFLAGS="$CFLAGS $LIBASS_CFLAGS" ./configure "$LIBASS_CONF" --enable-fuzz --enable-compare | |
| if [ -z "$SANITY" ] ; then | |
| make -j 2 | |
| else | |
| make CC="gcc -fsanitize=address -fsanitize=undefined -fsanitize=float-cast-overflow -fno-sanitize-recover=all" -j 2 | |
| fi | |
| ' > build-libass.sh | |
| # Test libass | |
| echo '#!/bin/sh | |
| . ./env.sh | |
| cd libass-tests | |
| ./run.sh ../libass/compare/compare ../libass/fuzz/fuzz | |
| ' > test-libass.sh | |
| - name: Mount Chroot | |
| run: | | |
| sudo sh -c " | |
| mount --rbind /sys/ $CHR_DIR/sys ; | |
| mount --rbind /proc/ $CHR_DIR/proc ; | |
| mount --rbind /dev/ $CHR_DIR/dev ; | |
| " | |
| - name: Chroot - Build Libass | |
| run: | | |
| sudo $OPRE chroot "$CHR_DIR" $IPRE /bin/dash -c ' | |
| su -c '\''/bin/sh -c "cd ~/workarea; dash ./build-libass.sh"'\'' --login artci | |
| ' | |
| - name: Chroot - Regression Tests | |
| run: | | |
| sudo $OPRE chroot "$CHR_DIR" $IPRE /bin/dash -c ' | |
| su -c '\''/bin/sh -c "cd ~/workarea; dash ./test-libass.sh"'\'' --login artci | |
| ' | |
| - name: Umount Chroot | |
| run: | | |
| sudo sh -c ' | |
| for m in sys proc dev ; do | |
| mount --make-rslave '"$CHR_DIR"'/$m | |
| umount -R '"$CHR_DIR"'/$m | |
| done | |
| ' |