build-test-release.yml

name: CI
on:
  push:
    branches:
      - main
    tags:
      - "*"
  pull_request:
  workflow_dispatch:
jobs:
  lint:
    name: Linting/style checks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: '.python-version'
          check-latest: true

      - name: Upgrade `pip` and install `wheel`
        run: python -m pip install --upgrade pip wheel

      - name: Install pre-commit
        run: pip install pre-commit

      - name: Cache pre-commit hooks
        uses: actions/cache@v3
        with:
          path: ~/.cache/pre-commit/
          key: pre-commit-cache|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}|${{ hashFiles('setup.cfg') }}

      - name: Run pre-commit checks
        run: pre-commit run --all-files --verbose

  type-check:
    name: Type-checking
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: '.python-version'
          check-latest: true

      - name: Upgrade `pip` and install `wheel`
        run: python -m pip install --upgrade pip wheel

      - name: Install tox
        run: pip install tox

      - name: Run type checking
        run: tox -e type

  # check-logic:
  #   name: Validate logic
  #   runs-on: ubuntu-latest
  #   steps:
  #     - uses: actions/checkout@v3

  #     - name: Checkout rando-shuffler repo
  #       uses: actions/checkout@v3
  #       with:
  #         repository: minishmaker/rando-shuffler
  #         path: rando-shuffler

  #     - name: Get cache key
  #       id: shuffler-cache-key
  #       run: echo "key=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
  #       shell: bash
  #       working-directory: rando-shuffler

  #     - uses: actions/cache@v3
  #       id: rando-shuffler-cache
  #       with:
  #         path: rando-shuffler
  #         key: ${{ steps.shuffler-cache-key.outputs.key }}

  #     - name: Compile rando-shuffler
  #       if: steps.rando-shuffler-cache.outputs.cache-hit != 'true'
  #       run: cargo build
  #       working-directory: rando-shuffler
  #       env:
  #         CARGO_TERM_COLOR: always

  #     - name: Make it executable
  #       run: chmod +x rando-shuffler/target/debug/rando_shuffler

  #     - name: Run the MCR shuffler
  #       run: |
  #         shopt -s globstar
  #         find rando-shuffler/ -name "*.logic" -type f -delete # remove test files
  #         file_count=0
  #         failed=0;
  #         for file in **/*.logic; do
  #           echo "$file... ";
  #           status=0
  #           tmpfile=$(mktemp /tmp/output.XXXXXX)  # create temporary file to store output
  #           rando-shuffler/target/debug/rando_shuffler "$file" >> $tmpfile 2>&1 || status=1
  #           if [[ $status != 0 ]]; then
  #             failed=$((failed + 1));
  #             echo -e "\033[0;31mFAILED\033[0m"
  #             cat $tmpfile  # print output of rando-shuffler if errors occured
  #           else
  #             echo -e "\033[0;32mPASSED\033[0m"
  #           fi
  #           echo "-------------------------------"
  #           echo ""
  #           file_count=$((file_count + 1))
  #         done;
  #         echo "RESULTS: $((file_count - failed)) / $file_count passed"
  #         if [[ $failed != 0 ]]; then
  #           exit 1;
  #         fi

  check-aux-data:
    name: Validate aux data
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: '.python-version'
          check-latest: true

      - name: Upgrade `pip` and install `wheel`
        run: python -m pip install --upgrade pip wheel

      - name: Install dependencies
        run: pip install -e .

      - name: Check if there are changes to pydantic models that aren't reflected in JSON schema
        run: |
          python ph_rando/shuffler/aux_models.py
          git diff --exit-code
          if [ $? -ne 0 ]
          then
            exit 1
          fi

      - name: Run pydantic validation on aux data
        run: |
          python <<HEREDOC
          import json, sys
          from pathlib import Path
          from pydantic import ValidationError
          from ph_rando.shuffler.aux_models import Area
          failed = False
          for file in (Path.cwd() / 'ph_rando' / 'shuffler' / 'logic').rglob('*.json'):
            with open(file) as f:
              try:
                Area(**json.load(f))
              except ValidationError as e:
                failed = True
                print(file.relative_to(Path.cwd()), file=sys.stderr, end='\n')
                print(e, file=sys.stderr, end='\n\n\n')
          if failed:
            exit(1)
          HEREDOC

  check-randomizer-settings:
    name: Validate randomizer settings
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: '.python-version'
          check-latest: true

      - name: Upgrade `pip` and install `wheel`
        run: python -m pip install --upgrade pip wheel

      - name: Install dependencies
        run: pip install -e .

      - name: Check if there are changes to pydantic models that aren't reflected in JSON schema
        run: |
          python ph_rando/settings.py
          git diff --exit-code
          if [ $? -ne 0 ]
          then
            exit 1
          fi

      - name: Run pydantic validation on settings json
        run: |
          python <<HEREDOC
          import json, sys
          from pathlib import Path
          from pydantic import ValidationError
          from ph_rando.settings import Settings
          failed = False
          with open(Path.cwd() / 'ph_rando' / 'settings.json') as f:
            try:
              Settings(**json.load(f))
            except ValidationError as e:
              failed = True
              print(file.relative_to(Path.cwd()), file=sys.stderr, end='\n')
              print(e, file=sys.stderr, end='\n\n\n')
          if failed:
            exit(1)
          HEREDOC

  run-shuffler:
    name: Run the shuffler against logic and aux data
    needs:
      - check-aux-data
      # - check-logic
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: '.python-version'
          check-latest: true

      - name: Upgrade `pip` and install `wheel`
        run: python -m pip install --upgrade pip wheel

      - name: Install dependencies
        run: pip install -e .

      - name: Run the shuffler
        run: ph_rando_shuffler -o -- --spoiler-log spoiler.txt

      - name: Print spoiler log
        run: cat spoiler.txt

      - name: Save spoiler log as an artifact
        uses: actions/upload-artifact@v3
        with:
          name: spoiler.txt
          path: spoiler.txt

  tests:
    name: Run tests
    needs:
      - check-aux-data
      # - check-logic
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu, macos, windows]
    defaults:
      run:
        shell: bash
    runs-on: ${{ matrix.os }}-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: '.python-version'
          check-latest: true

      - name: Upgrade `pip` and install `wheel`
        run: python -m pip install --upgrade pip wheel

      - name: Install tox
        run: pip install tox

      - name: Run tests
        run: tox -e test


  generate-base-patch:
    name: Build base patch [${{ matrix.rom_variant }}]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - rom_variant: US
            google_drive_url_secret: PH_US_GOOGLE_DRIVE_ID
          - rom_variant: US (D-Pad Patch)
            google_drive_url_secret: PH_US_DPAD_GOOGLE_DRIVE_ID
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: '.python-version'
          check-latest: true

      - name: Upgrade `pip` and install `wheel`
        run: python -m pip install --upgrade pip wheel

      - name: Fetch PH rom
        run: |
          pip install gdown
          gdown ${{ secrets[matrix.google_drive_url_secret] }}
        working-directory: base

      - name: Get filename of rom
        id: rom-filename
        run: echo "rom_filename=$(ls *.nds)" >> $GITHUB_OUTPUT
        working-directory: base

      - name: Get checksum of ROM
        id: rom-checksum
        run: echo "checksum=$(sha256sum '${{ steps.rom-filename.outputs.rom_filename }}' | awk '{ print $1 }')" >> $GITHUB_OUTPUT
        working-directory: base

      - name: Get current commit hash
        id: commit-hash
        run: echo "commit_hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3

      - name: Build base patch
        uses: docker/build-push-action@v6
        with:
          context: base/
          push: false
          # cache-from: type=gha
          # cache-to: type=gha,mode=max
          build-args: |
            PH_ROM_PATH=${{ steps.rom-filename.outputs.rom_filename }}
            VERSION_STRING=${{ steps.commit-hash.outputs.commit_hash }}
          outputs: |
            base/out

      - name: Save base patch as an artifact
        uses: actions/upload-artifact@v3
        with:
          name: ${{ steps.rom-checksum.outputs.checksum }}.bps
          path: base/out/${{ steps.rom-checksum.outputs.checksum }}.bps

      - name: Cleanup
        if: always()
        run: rm -rf *.nds

  generate-base-patch-test-jobs:
    name: Generate base patch test jobs
    runs-on: ubuntu-latest
    outputs:
      should-run-tests: ${{ steps.filter.outputs.base-patches }}
      job-matrix: ${{ steps.set-test-matrix.outputs.matrix }}
    steps:
    - uses: actions/checkout@v4

    - name: Check if base patch tests should be run
      uses: dorny/paths-filter@v3
      id: filter
      with:
        filters: |
          base-patches:
            - 'base/**'
            - 'tests/desmume/**'
            - '.github/workflows/**'
            - '.python-version'
            - 'setup.cfg'

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version-file: '.python-version'
        check-latest: true

    - name: Upgrade `pip` and install `wheel`
      run: python -m pip install --upgrade pip wheel

    - name: Install dependencies
      run: |
        pip install -e .[test,desmume]

    - name: Get list of tests to run in separate jobs
      id: set-test-matrix
      run: |
        python .github/generate_desmume_test_jobs.py
        echo "matrix=$(cat .github/matrix.json)" >> $GITHUB_OUTPUT

  test-desmume:
    name: Run base patch tests [${{ matrix.module }}] (${{ matrix.rom_variant }})
    needs:
      - generate-base-patch
      - generate-base-patch-test-jobs
    # Only run these tests if base patches or the tests themselves are modified, or if a release is happening, or on push to main.
    if: "startsWith(github.ref, 'refs/tags/') || needs.generate-base-patch-test-jobs.outputs.should-run-tests == 'true'"
    # Windows is used to run the desmume-based tests because py-desmume has issues running on headless ubuntu
    runs-on: windows-latest
    defaults:
      run:
        shell: bash
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.generate-base-patch-test-jobs.outputs.job-matrix) }}
    steps:
      - uses: actions/checkout@v4
        with:
          # Also retrieve previous commit so we can check if there are relevant changes
          fetch-depth: 2

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: '.python-version'
          check-latest: true

      - name: Upgrade `pip` and install `wheel`
        run: python -m pip install --upgrade pip wheel

      - name: Install system dependencies
        run: |
          choco install -y tesseract
          echo "C:\Program Files\Tesseract-OCR" >> $GITHUB_PATH

      - name: Fetch PH rom
        run: |
          pip install gdown
          gdown ${{ secrets[matrix.google_drive_url_secret] }}

      - name: Get filename of rom
        id: rom-filename
        run: echo "filename=$(ls *.nds)" >> $GITHUB_OUTPUT

      - name: Get checksum of ROM
        id: rom-checksum
        run: echo "checksum=$(sha256sum '${{ steps.rom-filename.outputs.filename }}' | awk '{ print $1 }')" >> $GITHUB_OUTPUT

      - name: Download base patch from previous workflow
        uses: actions/download-artifact@v3
        with:
          name: ${{ steps.rom-checksum.outputs.checksum }}.bps

      - name: Move base patch to directory expected by patcher
        run: |
          mkdir -p base/out
          mv ${{ steps.rom-checksum.outputs.checksum }}.bps base/out/${{ steps.rom-checksum.outputs.checksum }}.bps

      - name: Install tox
        run: pip install tox

      - name: Run tests
        run: PY_DESMUME_BATTERY_DIR="$pythonLocation" tox -e test-desmume -- -k "${{ matrix.module }}"
        env:
          PH_ROM_PATH: ${{ steps.rom-filename.outputs.filename }}
          PY_DESMUME_VIDEO_RECORDING_DIR: recordings/

      - name: Save recordings of failed tests as artifacts
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: test_recordings_${{ matrix.module }}_${{ matrix.rom_variant }}
          path: recordings/

      - name: Cleanup
        if: always()
        run: rm -rf *.nds

  build:
    name: Build randomizer
    needs:
      - generate-base-patch
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu, macos, windows]
    defaults:
      run:
        shell: bash
    runs-on: ${{ matrix.os }}-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: '.python-version'
          check-latest: true

      - name: Upgrade `pip` and install `wheel`
        run: python -m pip install --upgrade pip wheel

      - name: Install tox
        run: pip install tox

      - name: Download base patches
        uses: actions/download-artifact@v3

      - name: Move all patch files into the same directory so pyinstaller can find them
        run: |
          mkdir -p base/out
          mv ./**/*.bps base/out

      - name: Build randomizer executable
        run: tox -e build

      - name: Get current commit hash
        id: commit-hash
        run: echo "hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT

      - uses: actions/upload-artifact@v3
        with:
          name: ph-randomizer_${{ steps.commit-hash.outputs.hash }}_${{ matrix.os }}
          path: dist/

  run-executable:
    name: Run built randomizer executable [${{ matrix.os }} / ${{ matrix.rom_variant }}]
    needs: [build]
    runs-on: ${{ matrix.os }}-latest
    timeout-minutes: 10
    defaults:
      run:
        shell: bash
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu, macos, windows]
        rom_variant: [US, US (D-Pad Patch)]
        include:
          - rom_variant: US
            google_drive_url_secret: PH_US_GOOGLE_DRIVE_ID
          - rom_variant: US (D-Pad Patch)
            google_drive_url_secret: PH_US_DPAD_GOOGLE_DRIVE_ID
    steps:
      - uses: actions/checkout@v4

      # Note: intentionally use an older version of Python here;
      # The pyinstaller build should not depend on a specific Python
      # version being installed (or, Python being installed at all
      # for that matter).
      - name: Get two versions of Python behind min required
        id: python-version
        run: |
          PYTHON_VERSION=$(cat .python-version)
          echo "python-version="3.$(echo "${PYTHON_VERSION:2} 2" | awk '{print $1-$2}')"" >> $GITHUB_OUTPUT
          rm -rf *

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ steps.python-version.outputs.python-version }}

      - name: Upgrade `pip` and install `wheel`
        run: python -m pip install --upgrade pip wheel

      - name: Fetch PH rom
        run: |
          pip install gdown
          gdown ${{ secrets[matrix.google_drive_url_secret] }}

      - name: Get filename of rom
        id: rom-filename
        run: echo "rom_filename=$(ls *.nds)" >> $GITHUB_OUTPUT

      - name: Get current commit hash
        id: commit-hash
        run: echo "hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT

      - name: Download built executable from previous job
        uses: actions/download-artifact@v3
        with:
          name: ph-randomizer_${{ steps.commit-hash.outputs.hash }}_${{ matrix.os }}

      - name: Run executable (no GUI)
        run: |
          chmod +x ./ph_rando
          ./ph_rando --no-gui --input-rom-path="${{ steps.rom-filename.outputs.rom_filename }}" --output-rom-path="out.nds"

      - name: Cleanup
        if: always()
        run: rm -rf *.nds

  release:
    name: Publish a release
    # Require all tests to pass before a release is possible
    needs:
      - build
      - lint
      - type-check
      - tests
      # - check-logic
      - check-aux-data
      - run-shuffler
      - test-desmume
      - run-executable
    # Only create a release if a tag was pushed
    if: "startsWith(github.ref, 'refs/tags/')"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Get current commit hash
        id: commit-hash
        run: echo "hash=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT

      - name: Get version string
        id: version-string
        run: echo "version=$(git describe --tags)" >> $GITHUB_OUTPUT

      - name: Download built executables
        uses: actions/download-artifact@v3

      - name: Replace commit hash with version string
        run: |
          mv ph-randomizer_${{ steps.commit-hash.outputs.hash }}_ubuntu ph-randomizer_${{ steps.version-string.outputs.version }}_ubuntu
          mv ph-randomizer_${{ steps.commit-hash.outputs.hash }}_macos ph-randomizer_${{ steps.version-string.outputs.version }}_macos
          mv ph-randomizer_${{ steps.commit-hash.outputs.hash }}_windows ph-randomizer_${{ steps.version-string.outputs.version }}_windows

      - name: Zip up executables
        run: |
          zip -r -j ph-randomizer_${{ steps.version-string.outputs.version }}_linux.zip ph-randomizer_${{ steps.version-string.outputs.version }}_ubuntu
          zip -r -j ph-randomizer_${{ steps.version-string.outputs.version }}_macos.zip ph-randomizer_${{ steps.version-string.outputs.version }}_macos
          zip -r -j ph-randomizer_${{ steps.version-string.outputs.version }}_windows.zip ph-randomizer_${{ steps.version-string.outputs.version }}_windows

      - name: Create a GitHub Release
        uses: softprops/action-gh-release@v1
        with:
          files: |
            ph-randomizer_${{ steps.version-string.outputs.version }}_linux.zip
            ph-randomizer_${{ steps.version-string.outputs.version }}_macos.zip
            ph-randomizer_${{ steps.version-string.outputs.version }}_windows.zip