MAINT: Update documentation and logging (#155)

* MAINT: Update documentation and logging

- Nice new landing page
- Added covenience makefile commands
- improved logging of flagged epochs

* FIX, MAINT: Bump iclabel version and ignore mne warning

We already throw an error if the user has mne iclabel version less than .5, so added a pin for minimum version of .5 in the requirements file. MNE is also throwing a future warnign for an API kwarg change in epochs.get_data, where the current default copy=False is changing to copy=True. See mne-tools/mne-python#12121 . I'm a little worried about the additional copies we will incur every single time that we call that method, but I dont have a strong reason to stray from what MNE believes is a bug fix so lets go with it and see how it plays.

* FIX, MAINT: Bumpg GH actions checkout versions

- This should resolve the core issue of the runner not being able to find the most up to date
mne_iclabel wheel.

* FIX, TST: Install torch in doc building CI

Since MNE iclabel 0.5, torch is NOT installed by default.

.github/workflows/build_doc.yml

@@ -13,19 +13,24 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Install Pylossless
-        run: pip install -e .
+        run: |
+          pip install --upgrade pip
+          pip install -e .
 
       - name: install openneuro-py
         run: pip install openneuro-py
 
+      - name: install pytorch
+        run: pip install torch
+
       - name: Install doc dependencies
         run: pip install -r docs/requirements_doc.txt
 

docs/Makefile

@@ -12,6 +12,12 @@ clean:
 	-rm -rf $(BUILDDIR)/*
 	-rm -rf $(SOURCEDIR)/auto_examples/
 
+html-noplot:
+	$(SPHINXBUILD) -D plot_gallery=0 -b html $(SOURCEDIR) $(BUILDDIR)/html
+
+show:
+	open $(BUILDDIR)/html/index.html
+
 # Put it first so that "make" without argument is like "make help".
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/source/index.rst

@@ -1,66 +1,49 @@
-.. pyLossless documentation master file, created by
-   sphinx-quickstart on Fri Jan  6 12:24:18 2023.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
-PyLossless EEG Processing Pipeline
-==================================
-
-.. toctree::
-   :maxdepth: 1
-   :hidden:
+:layout: landing
+:description: Shibuya is a modern, responsive, customizable theme for Sphinx.
 
-   install
-   auto_examples/index.rst
-   API/API_index
-   contributing
+PyLossless :octicon:`pulse`
+===========================
 
+.. rst-class:: lead
 
+    EEG Processing Pipeline that is non-destructive, automated, and built on Python.
 
-.. grid::
-
-   .. grid-item-card::
+.. container:: buttons
 
-      |:mechanical_arm:| Automated
-      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-      Automatic Processing Pipeline
-      cleans your EEG data. |:broom:|
+    `Docs <install.html>`_
+    `GitHub <https://github.com/lina-usc/pylossless>`_
 
 
-   .. grid-item-card::
+.. grid:: 1 1 2 3
+    :gutter: 2
+    :padding: 0
+    :class-row: surface
 
-      |:snake:| Built on Python
-      ^^^^^^^^^^^^^^^^^^^^^^^^^
-      Ported from MATLAB for easier
-      use and access!
+    .. grid-item-card:: :octicon:`zap` Automated
 
-   .. grid-item-card::
+        Fast, Open-source, and built on python.
 
-      |:recycle:| Non-destructive
-      ^^^^^^^^^^^^^^^^^^^^^^^^^^^
-      Keeps your EEG continuous, so
-      you can epoch your data however
-      and whenever you want to.
+    .. grid-item-card:: :octicon:`pin` Non-destructive
 
-.. grid::
+        Keeps EEG continuous, noting bad channels, times, and independent components
+        so you can reject them and epoch your data however and whenever you want to.
 
-   .. grid-item-card:: 
+    .. grid-item-card:: :octicon:`telescope-fill` Streamlined Review
 
-      |:pencil:| Artifacts are Noted
-      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-      Bad channels, times, and
-      components are stored as
-      ``Annotations`` in your
-      raw data. 
+         Web dashboard built with helps you review the output and make
+         informed decisions about your data.
 
-   .. grid-item-card::
-
-      |:woman_technologist:| Streamlined Review
-      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-      Web dashboard built with Plotly/Dash helps
-      you Review the pipeline output and make
-      informed decisions about your data
-
 .. image:: https://raw.githubusercontent.com/scott-huberty/wip_pipeline-figures/main/dashboard.png
    :alt: pylossless-qc-dashboard-screenshot
-   :align: center
+   :align: center
+
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+
+   install
+   auto_examples/index.rst
+   API/API_index
+   contributing
+   Paper <https://www.biorxiv.org/content/10.1101/2024.01.12.575323v1>

examples/README.txt

@@ -1,2 +1,2 @@
-PyLossless Tutorials
-^^^^^^^^^^^^^^^^^^^^
+Tutorials
+^^^^^^^^^

pylossless/_logging.py

@@ -43,7 +43,7 @@ def wrapper(*args, message=None, **kwargs):
             logger.info(f"LOSSLESS: Skipping {this_step}")
             return
         elif verbose:
-            logger.info(f"LOSSLESS: 🚩 {this_step}.")
+            logger.info(f"LOSSLESS: 👇 {this_step}.")
         result = func(*args, **kwargs)
         end_time = time.time()
         dur = f"{end_time - start_time:.2f}"

pylossless/conftest.py

@@ -13,6 +13,9 @@
 
 import pytest
 
+# TODO: Remove this once mne 1.7 is released
+pytest.mark.filterwarnings("ignore:The current default of copy")
+
 
 @pytest.fixture(scope="session")
 def pipeline_fixture():

pylossless/flagging.py

@@ -197,9 +197,8 @@ def add_flag_cat(self, kind, bad_epoch_inds, epochs):
     def load_from_raw(self, raw):
         """Load pylossless annotations from raw object."""
         sfreq = raw.info["sfreq"]
-        lossless_descriptions = ["bad_noisy", "bad_uncorrelated", "bad_noisy_ICs"]
         for annot in raw.annotations:
-            if annot["description"] in lossless_descriptions:
+            if annot["description"].upper().startswith("BAD_LL"):
                 ind_onset = int(np.round(annot["onset"] * sfreq))
                 ind_dur = int(np.round(annot["duration"] * sfreq))
                 inds = np.arange(ind_onset, ind_onset + ind_dur)

pylossless/pipeline.py

@@ -31,6 +31,7 @@
 from .config import Config
 from .flagging import FlaggedChs, FlaggedEpochs, FlaggedICs
 from ._logging import lossless_logger, lossless_time
+from .utils import _report_flagged_epochs
 from .utils.html import _get_ics, _sum_flagged_times, _create_html_details
 
 
@@ -476,9 +477,9 @@ def _repr_html_(self):
         channel_noise = _get_ics(df, "channel_noise")
 
         lossless_flags = [
-            "bad_noisy",
-            "bad_uncorrelated",
-            "bad_noisy_ICs",
+            "BAD_LL_noisy",
+            "BAD_LL_uncorrelated",
+            "BAD_LL_noisy_ICs",
         ]
         flagged_times = _sum_flagged_times(self.raw, lossless_flags)
 
@@ -579,13 +580,14 @@ def add_pylossless_annotations(self, inds, event_type, epochs):
         """
         # Concatenate epoched data back to continuous data
         t_onset = epochs.events[inds, 0] / epochs.info["sfreq"]
+        desc = f"BAD_LL_{event_type}"
         df = pd.DataFrame(t_onset, columns=["onset"])
         # We exclude the last sample from the duration because
         # if the annot lasts the whole duration of the epoch
         # it's end will coincide with the first sample of the
         # next epoch, causing it to erroneously be rejected.
         df["duration"] = 1 / epochs.info["sfreq"] * len(epochs.times[:-1])
-        df["description"] = f"bad_{event_type}"
+        df["description"] = desc
 
         # Merge close onsets to prevent a bunch of 1-second annotations of the same name
         # find onsets close enough to be considered the same
@@ -604,6 +606,7 @@ def add_pylossless_annotations(self, inds, event_type, epochs):
             orig_time=self.raw.annotations.orig_time,
         )
         self.raw.set_annotations(self.raw.annotations + annotations)
+        _report_flagged_epochs(self.raw, desc)
 
     def get_events(self):
         """Make an MNE events array of fixed length events."""
@@ -865,7 +868,6 @@ def flag_noisy_epochs(self):
         bad_epoch_inds = _detect_outliers(
             data_sd, flag_dim="epoch", init_dir="pos", **config_epoch
         )
-        logger.info(f"📋 LOSSLESS: Noisy epochs: {bad_epoch_inds}")
         self.flags["epoch"].add_flag_cat("noisy", bad_epoch_inds, epochs)
 
     def get_n_nbr(self):
@@ -976,7 +978,6 @@ def flag_uncorrelated_epochs(self):
             init_dir="neg",
             **self.config["uncorrelated_epochs"],
         )
-        logger.info(f"📋 LOSSLESS: Uncorrelated epochs: {bad_epoch_inds}")
         self.flags["epoch"].add_flag_cat("uncorrelated", bad_epoch_inds, epochs)
 
     @lossless_logger
@@ -1175,7 +1176,7 @@ def _run(self):
         self.flag_noisy_ics(message="Flagging time periods with noisy IC's.")
 
         # 12. TODO: integrate labels from IClabels to self.flags["ic"]
-        self.run_ica("run2", message="Running Final ICA.")
+        self.run_ica("run2", message="Running Final ICA and ICLabel.")
 
     def run_dataset(self, paths):
         """Run a full dataset.

pylossless/utils/__init__.py

@@ -1,2 +1,2 @@
-from ._utils import _icalabel_to_data_frame
+from ._utils import _icalabel_to_data_frame, _report_flagged_epochs
 from .html import _get_ics, _sum_flagged_times, _create_html_details

pylossless/utils/_utils.py

@@ -7,7 +7,11 @@
 
 """Utility Functions for running the Lossless Pipeline."""
 
+import numpy as np
 import pandas as pd
+from mne.utils import logger
+
+from .html import _sum_flagged_times
 
 
 def _icalabel_to_data_frame(ica):
@@ -26,3 +30,11 @@ def _icalabel_to_data_frame(ica):
             confidence=ica.labels_scores_.max(1),
         )
     )
+
+
+def _report_flagged_epochs(raw, flag):
+    times = _sum_flagged_times(raw, flag)[flag]
+    if not times:
+        times = 0
+    else:
+        logger.info(f"📋 LOSSLESS: {np.round(times, 2)} second(s) flagged as {flag}")

pylossless/utils/html.py

@@ -10,6 +10,8 @@ def _get_ics(df, ic_type):
 def _sum_flagged_times(raw, flags):
     """Sum the total time flagged for various flags like noisy etc."""
     flag_dict = {}
+    if isinstance(flags, str):
+        flags = [flags]
     for flag in flags:
         flag_dict[flag] = []
         if raw:

requirements.txt

@@ -5,6 +5,6 @@ mne_bids
 pandas
 xarray
 scipy>=1.2.1
-mne_icalabel
+mne_icalabel>=0.5.0
 pyyaml
 scikit-learn