wip: new baselines

.github/workflows/regression_tests.yml

@@ -0,0 +1,39 @@
+# .github/workflows/regression_tests.yml
+name: Regression Tests
+
+on:
+  # pull_request:
+  #   branches:
+  #     - main
+
+jobs:
+  regression_tests:
+    name: regression_tests
+    runs-on: ubuntu-20.04
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        lfs: true
+        fetch-depth: 0 # This ensures we can checkout main branch too
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+        architecture: 'x64'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e ".[dev]"
+       
+    - name: Run benchmarks and compare to baseline
+      if: github.event.pull_request.base.ref == 'main'
+      run: |
+        # Check if regression test results exist in main branch
+        if [ -f 'git cat-file -e main:tests/regression_test_baselines.json' ]; then
+          git checkout main tests/regression_test_baselines.json
+        else
+          echo "No regression test results found in main branch"
+        fi
+        pytest -m regression

.github/workflows/run_tutorials.yml

@@ -0,0 +1,42 @@
+name: Run Tutorials
+on:
+  push:
+    branches:
+      - main
+  release:
+    types: [ published ]
+
+jobs:
+  build:
+    name: Run Tutorials
+    runs-on: ubuntu-20.04
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        lfs: true
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+        architecture: 'x64'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e ".[dev]"
+
+    - name: Modify notebook parameters
+      run: |
+        # Replace parameters to reduce exec time of 07_gradient_descent
+        sed -i 's/batch_size = 4/batch_size = 1/' docs/tutorials/07_gradient_descent.ipynb
+        sed -i 's/for epoch in range(10):/for epoch in range(1):/' docs/tutorials/07_gradient_descent.ipynb
+        sed -i 's/inputs = jnp.asarray(np.random.rand(100, 2))/inputs = jnp.asarray(np.random.rand(3, 2))/' docs/tutorials/07_gradient_descent.ipynb
+        sed -i 's/t_max = 5.0/t_max = 3.0/' docs/tutorials/07_gradient_descent.ipynb
+
+    - name: Test notebooks
+      run: |
+        for notebook in docs/tutorials/*.ipynb; do
+          echo "Testing $notebook"
+          jupyter nbconvert --to notebook --execute --ExecutePreprocessor.timeout=600 "$notebook"
+        done

.github/workflows/tests.yml

@@ -1,12 +1,12 @@
 name: Tests
 
 on:
-  push:
-    branches:
-    - main
-  pull_request:
-    branches:
-      - main
+#   push:
+#     branches:
+#     - main
+#   pull_request:
+#     branches:
+#       - main
 
 jobs:
   build:
@@ -39,4 +39,4 @@ jobs:
     - name: Test with pytest
       run: |
         pip install pytest pytest-cov
-        pytest tests/ --cov=jaxley --cov-report=xml
+        pytest tests/ -m "not regression" --cov=jaxley --cov-report=xml

.github/workflows/update_regression_baseline.yml

@@ -0,0 +1,90 @@
+# .github/workflows/update_regression_tests.yml
+
+# for details on triggering a workflow from a comment, see:
+# https://dev.to/zirkelc/trigger-github-workflow-for-comment-on-pull-request-45l2
+name: Update Regression Baseline
+
+on:
+  issue_comment: # trigger from comment; event runs on the default branch
+    types: [created]
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  update_regression_tests:
+    name: update_regression_tests
+    runs-on: ubuntu-20.04
+    # Trigger from a comment
+    # if: github.event.issue.pull_request && contains(github.event.comment.body, '/update_baseline')
+    permissions:
+      contents: write
+      pull-requests: write
+    env:
+      username: ${{ github.event.pull_request.user.login }} # ${{ github.actor }}
+
+    steps:
+    - name: Get PR branch
+      uses: xt0rted/pull-request-comment-branch@v1
+      id: comment-branch
+
+    - name: Checkout PR branch
+      uses: actions/checkout@v3
+      with:
+        # ref: ${{ steps.comment-branch.outputs.head_sha }} # using head_sha vs. head_ref makes this work for forks
+        lfs: true
+        fetch-depth: 0 # This ensures we can checkout main branch too
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+        architecture: 'x64'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e ".[dev]"
+       
+    - name: Update baseline
+      if: github.event.pull_request.base.ref == 'main'
+      run: |
+        git config --global user.name '$username'
+        git config --global user.email '[email protected]'
+        # Check if regression test results exist in main branch
+        if [ -f 'git cat-file -e main:tests/regression_test_baselines.json' ]; then
+          git checkout main tests/regression_test_baselines.json
+        else
+          echo "No regression test results found in main branch"
+        fi
+        NEW_BASELINE=1 pytest -m regression
+
+    - name: Add Baseline update report to PR comment
+      uses: actions/github-script@v7
+      if: github.event.pull_request.base.ref == 'main' # might need `always()` to work
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          const fs = require('fs');
+          const TestReport = fs.readFileSync('tests/regression_test_report.txt', 'utf8');
+          
+          await github.rest.issues.createComment({
+            issue_number: context.issue.number,
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: `## New Baselines \n\`\`\`\n${TestReport}\n\`\`\``
+          });
+                
+    - name: Commit and push
+      if: github.event.pull_request.base.ref == 'main'
+      run: |
+        git add -f tests/regression_test_baselines.json # since it's in .gitignore
+        git commit -m "Update regression test baselines"
+        git push origin HEAD:${{ github.head_ref }}
+    
+    # Needed when workflow is triggered from a comment
+    # - name: Commit and push
+    #   if: github.event.pull_request.base.ref == 'main'
+    #   run: |
+    #     git add -f tests/regression_test_baselines.json # since it's in .gitignore
+    #     git commit -m "Update regression test baselines"
+    #     git push origin HEAD:${{ steps.comment-branch.outputs.head_sha }}

.gitignore

@@ -55,6 +55,8 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+tests/regression_test_results.json
+tests/regression_test_baselines.json
 
 # Translations
 *.mo

docs/advanced_tutorials.rst

@@ -6,6 +6,7 @@ Advanced tutorials
 .. toctree::
    :maxdepth: 1
 
+   tutorials/00_jaxley_api.ipynb
    tutorials/08_importing_morphologies.ipynb
    tutorials/09_advanced_indexing.ipynb
    tutorials/10_advanced_parameter_sharing.ipynb

docs/index.rst

@@ -42,17 +42,7 @@ Getting started
    plt.plot(v.T)  # Plot voltage trace.
 
 
-If you want to learn more, we have tutorials on how to:
-
-- `simulate morphologically detailed neurons <notebooks/tutorials/01_morph_neurons.html>`_
-- `simulate networks of such neurons <notebooks/tutorials/02_small_network.html>`_
-- `set parameters of cells and networks <notebooks/tutorials/03_setting_parameters.html>`_
-- `speed up simulations with GPUs and jit <notebooks/tutorials/04_jit_and_vmap.html>`_
-- `define your own channels and synapses <notebooks/tutorials/05_channel_and_synapse_models.html>`_
-- `define groups <notebooks/tutorials/06_groups.html>`_
-- `read and handle SWC files <notebooks/tutorials/08_importing_morphologies.html>`_
-- `compute the gradient and train biophysical models <notebooks/tutorials/07_gradient_descent.html>`_
-
+If you want to learn more, check out our `Tutorial on the basics of Jaxley <https://jaxley.readthedocs.io/en/latest/tutorials/01_morph_neurons.html>`_. For more resources, see the `FAQ <https://jaxley.readthedocs.io/en/latest/faq.html>`_ or `Advanced tutorials <https://jaxley.readthedocs.io/en/latest/advanced_tutorials.html>`_.
 
 
 Installation

docs/reference/jaxley.connect.rst

@@ -1,7 +1,7 @@
 Connecting Cells
 ===============================
 
-.. automodule:: jaxley.connect.connect
+.. autofunction:: jaxley.connect.connect
 .. autofunction:: jaxley.connect.connectivity_matrix_connect
 .. autofunction:: jaxley.connect.fully_connect
 .. autofunction:: jaxley.connect.sparse_connect

docs/reference/jaxley.utils.rst

@@ -4,8 +4,6 @@ Utils
    :members:
 .. automodule:: jaxley.utils.plot_utils
    :members:
-.. automodule:: jaxley.utils.swc
-   :members:
 .. automodule:: jaxley.utils.jax_utils
    :members:
 .. automodule:: jaxley.utils.syn_utils

docs/tutorials.rst

@@ -8,7 +8,6 @@ Tutorials
 
    tutorials/01_morph_neurons.ipynb
    tutorials/02_small_network.ipynb
-   tutorials/03_setting_parameters.ipynb
    tutorials/04_jit_and_vmap.ipynb
    tutorials/05_channel_and_synapse_models.ipynb
    tutorials/07_gradient_descent.ipynb