ci.yml

name: ci

on:
  pull_request:
  push:
    branches:
      - main

defaults:
  run:
    shell: bash

env:
  dataset: ci_test
  # NOTE: if any of these versions are changed, update the POM files too
  java_version: 17
  java_distribution: zulu
  groovy_version: 4.x


concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:

  # build
  #############################################################################

  build_timelines:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: setup java
      uses: actions/setup-java@v4
      with:
        java-version: ${{ env.java_version }}
        distribution: ${{ env.java_distribution }}
    - name: build
      run: mvn package
    - name: tar # to preserve any permissions
      run: |
        tar cavf build_detectors.tar.zst detectors/target
        tar cavf build_monitoring.tar.zst monitoring/target
    - uses: actions/upload-artifact@v4
      with:
        name: build_timelines
        retention-days: 1
        path: build*.tar.zst

  build_coatjava:
    runs-on: ubuntu-latest
    steps:
    - name: setup java
      uses: actions/setup-java@v4
      with:
        java-version: ${{ env.java_version }}
        distribution: ${{ env.java_distribution }}
    - name: checkout coatjava
      uses: actions/checkout@v4
      with:
        repository: JeffersonLab/coatjava
        ref: development
    - name: build coatjava
      run: ./build-coatjava.sh
    - name: tree
      run: tree
    - name: tar
      run: tar cavf build_coatjava.tar.zst coatjava
    - uses: actions/upload-artifact@v4
      with:
        name: build_coatjava
        retention-days: 1
        path: build*.tar.zst

  # download test data
  #############################################################################

  download_test_data:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: true
      matrix:
        include:
          - { type: detectors, schema: mon }
          - { type: physics,   schema: dst }
    env:
      xrootd_file: xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/validation/recon/${{ matrix.schema }}/validation_files.tar.zst
    steps:
    - uses: actions/cache@v4
      id: cache
      with:
        key: validation_files_${{ matrix.type }}
        path: validation_files.tar.zst
        lookup-only: true
    - name: install dependencies
      if: ${{ steps.cache.outputs.cache-hit != 'true' }}
      run: |
        sudo apt -y update
        sudo apt -y upgrade
        sudo apt -y install xrootd-client
    - name: download
      if: ${{ steps.cache.outputs.cache-hit != 'true' }}
      run: xrdcp ${{ env.xrootd_file }} ./
    - run: ls -lh .
      if: ${{ steps.cache.outputs.cache-hit != 'true' }}

  # monitoring
  #############################################################################

  run_monitoring:
    needs:
      - build_timelines
      - build_coatjava
      - download_test_data
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        type: [ detectors, physics ]
    steps:
    - name: setup java
      uses: actions/setup-java@v4
      with:
        java-version: ${{ env.java_version }}
        distribution: ${{ env.java_distribution }}
    - name: setup groovy
      uses: wtfjoke/setup-groovy@v2
      with:
        groovy-version: ${{ env.groovy_version }}
    - name: groovy version
      run: groovy --version
    - uses: actions/checkout@v4
    - uses: actions/cache/restore@v4
      with:
        key: validation_files_${{ matrix.type }}
        path: validation_files.tar.zst
    - uses: actions/download-artifact@v4
      with:
        pattern: build_*
        merge-multiple: true
    - name: untar
      run: ls *.tar.zst | xargs -I{} tar xavf {}
    - name: tree
      run: tree
    - name: run monitoring
      run: bin/run-monitoring.sh -d ${{env.dataset}} --findhipo --series --focus-${{matrix.type}} validation_files
    - name: tree slurm
      run: tree slurm
    - name: tree outfiles
      run: tree outfiles
    - uses: actions/upload-artifact@v4
      with:
        name: slurm_${{ matrix.type }}
        retention-days: 1
        path: slurm
    - uses: actions/upload-artifact@v4
      with:
        name: outfiles_monitoring_${{ matrix.type }}
        retention-days: 14
        path: outfiles

  test_swifjob:
    needs:
      - build_timelines
      - build_coatjava
      - download_test_data
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        type: [ detectors, physics ]
    steps:
    - name: setup java
      uses: actions/setup-java@v4
      with:
        java-version: ${{ env.java_version }}
        distribution: ${{ env.java_distribution }}
    - name: setup groovy
      uses: wtfjoke/setup-groovy@v2
      with:
        groovy-version: ${{ env.groovy_version }}
    - name: groovy version
      run: groovy --version
    - uses: actions/checkout@v4
    - uses: actions/cache/restore@v4
      with:
        key: validation_files_${{ matrix.type }}
        path: validation_files.tar.zst
    - uses: actions/download-artifact@v4
      with:
        pattern: build_*
        merge-multiple: true
    - name: untar
      run: ls *.tar.zst | xargs -I{} tar xavf {}
    - name: tree
      run: tree
    - name: test monitoring swifjob
      run: |
        single_rundir=$(find validation_files -mindepth 1 -maxdepth 1 -type d | head -n1)
        echo "single_rundir = $single_rundir"
        bin/test-swifjob.sh $single_rundir runner --focus-${{ matrix.type }}
    - name: tree runner
      run: tree runner

  # timeline production
  #############################################################################

  run_timelines:
    needs:
      - run_monitoring
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - { type: detectors,  args: '-n 4 --skip-mya' }
          - { type: physics,    args: '' }
    steps:
    - name: setup java
      uses: actions/setup-java@v4
      with:
        java-version: ${{ env.java_version }}
        distribution: ${{ env.java_distribution }}
    - name: setup groovy
      uses: wtfjoke/setup-groovy@v2
      with:
        groovy-version: ${{ env.groovy_version }}
    - name: groovy version
      run: groovy --version
    - uses: actions/checkout@v4
    - uses: actions/download-artifact@v4
      with:
        pattern: outfiles_*
        merge-multiple: true
        path: outfiles
    - uses: actions/download-artifact@v4
      with:
        pattern: build_*
        merge-multiple: true
    - name: untar
      run: ls *.tar.zst | xargs -I{} tar xavf {}
    - name: tree
      run: tree
    - name: run timelines
      run: bin/run-${{matrix.type}}-timelines.sh -d ${{env.dataset}} ${{matrix.args}}
    - name: tree outfiles
      run: tree outfiles
    - uses: actions/upload-artifact@v4
      with:
        name: outfiles_timelines_${{ matrix.type }}
        retention-days: 14
        path: outfiles

  # timeline deployment
  #############################################################################

  deploy_timelines:
    name: Deployment
    needs:
      - run_timelines
    runs-on: ubuntu-latest
    steps:
    - name: setup groovy
      uses: wtfjoke/setup-groovy@v2
      with:
        groovy-version: ${{ env.groovy_version }}
    - name: groovy version
      run: groovy --version
    - uses: actions/checkout@v4
    - uses: actions/download-artifact@v4
      with:
        pattern: outfiles_*
        merge-multiple: true
        path: outfiles
    - uses: actions/download-artifact@v4
      with:
        pattern: build_*
        merge-multiple: true
    - name: untar
      run: ls *.tar.zst | xargs -I{} tar xavf {}
    - name: tree outfiles
      run: |
        rm -r outfiles/${{ env.dataset }}/log
        echo "# Output File Tree" >> $GITHUB_STEP_SUMMARY
        echo '```' >> $GITHUB_STEP_SUMMARY
        tree outfiles | xargs -0 -I{} echo {} >> $GITHUB_STEP_SUMMARY
        echo '```' >> $GITHUB_STEP_SUMMARY
    - name: deploy
      run: bin/deploy-timelines.sh -d ${{env.dataset}} -c -t web
    - uses: actions/upload-artifact@v4
      with:
        name: web
        retention-days: 14
        path: web