Merge 1be9922 into 11279a5

.github/workflows/regression_tests.yml

@@ -0,0 +1,40 @@
+# .github/workflows/regression_tests.yml
+name: Regression Tests
+
+on:
+  # pull_request:
+  #   branches:
+  #     - main
+
+jobs:
+  regression_tests:
+    name: regression_tests
+    runs-on: ubuntu-20.04
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        lfs: true
+        fetch-depth: 0 # This ensures we can checkout main branch too
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+        architecture: 'x64'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e ".[dev]"
+       
+    - name: Run benchmarks and compare to baseline
+      if: github.event.pull_request.base.ref == 'main'
+      run: |
+        # # Check if regression test results exist in main branch
+        # if [ -f 'git cat-file -e main:tests/regression_test_baselines.json' ]; then
+        #   git checkout main tests/regression_test_baselines.json
+        # else
+        #   echo "No regression test results found in main branch"
+        # fi
+        pytest -m regression
+        # git checkout

.github/workflows/tests.yml

@@ -1,12 +1,12 @@
 name: Tests
 
 on:
-  push:
-    branches:
-    - main
-  pull_request:
-    branches:
-      - main
+#   push:
+#     branches:
+#     - main
+#   pull_request:
+#     branches:
+#       - main
 
 jobs:
   build:
@@ -39,4 +39,4 @@ jobs:
     - name: Test with pytest
       run: |
         pip install pytest pytest-cov
-        pytest tests/ --cov=jaxley --cov-report=xml
+        pytest tests/ -m "not regression" --cov=jaxley --cov-report=xml

.github/workflows/update_regression_baseline.yml

@@ -0,0 +1,75 @@
+# .github/workflows/update_regression_tests.yml
+name: Update Regression Baseline
+
+on:
+  # issue_comment: # event runs on the default branch
+  #   types: [created]
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  update_regression_tests:
+    name: update_regression_tests
+    runs-on: ubuntu-20.04
+    # if: github.event.issue.pull_request && contains(github.event.comment.body, '/update_baseline')
+    permissions:
+      contents: write
+      pull-requests: write
+    env:
+      username: ${{ github.event.pull_request.user.login }} # ${{ github.actor }}
+
+    steps:
+    # - name: Get PR branch
+    #   uses: xt0rted/pull-request-comment-branch@v1
+    #   id: comment-branch
+
+    - name: Checkout PR branch
+      uses: actions/checkout@v3
+      with:
+        # ref: ${{ steps.comment-branch.outputs.head_sha }} # using head_sha vs. head_ref makes this work for forks
+        lfs: true
+        fetch-depth: 0 # This ensures we can checkout main branch too
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+        architecture: 'x64'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e ".[dev]"
+       
+    - name: Update baseline
+      if: github.event.pull_request.base.ref == 'main'
+      run: |
+        git config --global user.name '$username'
+        git config --global user.email '[email protected]'
+        mv tests/regression_test_baselines.json tests/regression_test_baselines.json.bak
+        NEW_BASELINE=1 pytest -m regression
+
+    - name: Add Baseline update report to PR comment
+      uses: actions/github-script@v7
+      if: always()
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          const fs = require('fs');
+          const TestReport = fs.readFileSync('tests/regression_test_report.txt', 'utf8');
+          
+          await github.rest.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: `## New Baselines \n\`\`\`\n${TestReport}\n\`\`\``
+          });
+        
+        
+    - name: Commit and push
+      if: github.event.pull_request.base.ref == 'main'
+      run: |
+        git add -f tests/regression_test_baselines.json # since it's in .gitignore
+        git commit -m "Update regression test baselines"
+        # git push origin HEAD:${{ steps.comment-branch.outputs.head_sha }}
+        git push origin HEAD:${{ github.head_ref }}

.gitignore

@@ -55,6 +55,8 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+tests/regression_test_results.json
+tests/regression_test_baselines.json
 
 # Translations
 *.mo

pyproject.toml

@@ -69,6 +69,7 @@ dev = [
 [tool.pytest.ini_options]
 markers = [
     "slow: marks tests as slow (T > 10s)",
+    "regression: marks regression tests",
 ]
 
 [tool.isort]

tests/conftest.py

@@ -1,6 +1,7 @@
 # This file is part of Jaxley, a differentiable neuroscience simulator. Jaxley is
 # licensed under the Apache License Version 2.0, see <https://www.apache.org/licenses/>
 
+import json
 import os
 from copy import deepcopy
 from typing import Optional
@@ -9,6 +10,7 @@
 
 import jaxley as jx
 from jaxley.synapses import IonotropicSynapse
+from tests.test_regression import generate_regression_report
 
 
 @pytest.fixture(scope="session")
@@ -202,3 +204,36 @@ def get_or_compute_swc2jaxley_params(
 
     yield get_or_compute_swc2jaxley_params
     params = {}
+
+
+@pytest.fixture(scope="session", autouse=True)
+def print_session_report(request):
+    """Cleanup a testing directory once we are finished."""
+
+    def print_regression_report():
+        dirname = os.path.dirname(__file__)
+        baseline_fname = os.path.join(dirname, "regression_test_baselines.json")
+        results_fname = os.path.join(dirname, "regression_test_results.json")
+
+        baselines = {}
+        if os.path.exists(baseline_fname):
+            with open(baseline_fname) as f:
+                baselines = json.load(f)
+
+        results = {}
+        if os.path.exists(results_fname):
+            with open(results_fname) as f:
+                results = json.load(f)
+
+        # the following allows to print the report to the console despite pytest
+        # capturing the output and without specifying the "-s" flag
+        capmanager = request.config.pluginmanager.getplugin("capturemanager")
+        with capmanager.global_and_fixture_disabled():
+            print("\n\n\nRegression Test Report\n----------------------\n")
+            if not baselines:
+                print(
+                    "No baselines found. Run `git checkout main;UPDATE_BASELINE=1 pytest -m regression; git checkout -`"
+                )
+            print(generate_regression_report(baselines, results))
+
+    request.addfinalizer(print_regression_report)

tests/regression_test_baselines.json

@@ -0,0 +1,17 @@
+{
+  "ec3a4fad11d2bfb1bc5f8f10529cb06f2ff9919b377e9c0a3419c7f7f237f06e": {
+    "test_name": "test_runtime",
+    "input_kwargs": {
+      "num_cells": 1,
+      "artificial": false,
+      "connect": false,
+      "connection_prob": 0.0,
+      "voltage_solver": "jaxley.stone"
+    },
+    "runtimes": {
+      "build_time": 0.10014088948567708,
+      "compile_time": 0.3103648026784261,
+      "run_time": 0.2102543512980143
+    }
+  }
+}