From c84108c5a5452743b5aa9f60d36017c62868d14c Mon Sep 17 00:00:00 2001 From: Scott Huberty <52462026+scott-huberty@users.noreply.github.com> Date: Thu, 18 Jan 2024 12:00:43 -0800 Subject: [PATCH] MAINT: Update documentation and logging (#155) * MAINT: Update documentation and logging - Nice new landing page - Added covenience makefile commands - improved logging of flagged epochs * FIX, MAINT: Bump iclabel version and ignore mne warning We already throw an error if the user has mne iclabel version less than .5, so added a pin for minimum version of .5 in the requirements file. MNE is also throwing a future warnign for an API kwarg change in epochs.get_data, where the current default copy=False is changing to copy=True. See https://github.com/mne-tools/mne-python/pull/12121 . I'm a little worried about the additional copies we will incur every single time that we call that method, but I dont have a strong reason to stray from what MNE believes is a bug fix so lets go with it and see how it plays. * FIX, MAINT: Bumpg GH actions checkout versions - This should resolve the core issue of the runner not being able to find the most up to date mne_iclabel wheel. * FIX, TST: Install torch in doc building CI Since MNE iclabel 0.5, torch is NOT installed by default. --- .github/workflows/build_doc.yml | 13 ++++-- docs/Makefile | 6 +++ docs/source/index.rst | 83 +++++++++++++-------------------- examples/README.txt | 4 +- pylossless/_logging.py | 2 +- pylossless/conftest.py | 3 ++ pylossless/flagging.py | 3 +- pylossless/pipeline.py | 15 +++--- pylossless/utils/__init__.py | 2 +- pylossless/utils/_utils.py | 12 +++++ pylossless/utils/html.py | 2 + requirements.txt | 2 +- 12 files changed, 79 insertions(+), 68 deletions(-) diff --git a/.github/workflows/build_doc.yml b/.github/workflows/build_doc.yml index f9076f5..88b4e8f 100644 --- a/.github/workflows/build_doc.yml +++ b/.github/workflows/build_doc.yml @@ -13,19 +13,24 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.9 - name: Install Pylossless - run: pip install -e . + run: | + pip install --upgrade pip + pip install -e . - name: install openneuro-py run: pip install openneuro-py + - name: install pytorch + run: pip install torch + - name: Install doc dependencies run: pip install -r docs/requirements_doc.txt diff --git a/docs/Makefile b/docs/Makefile index e34a181..f329335 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -12,6 +12,12 @@ clean: -rm -rf $(BUILDDIR)/* -rm -rf $(SOURCEDIR)/auto_examples/ +html-noplot: + $(SPHINXBUILD) -D plot_gallery=0 -b html $(SOURCEDIR) $(BUILDDIR)/html + +show: + open $(BUILDDIR)/html/index.html + # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/source/index.rst b/docs/source/index.rst index 3966fb6..71ac406 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,66 +1,49 @@ -.. pyLossless documentation master file, created by - sphinx-quickstart on Fri Jan 6 12:24:18 2023. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -PyLossless EEG Processing Pipeline -================================== - -.. toctree:: - :maxdepth: 1 - :hidden: +:layout: landing +:description: Shibuya is a modern, responsive, customizable theme for Sphinx. - install - auto_examples/index.rst - API/API_index - contributing +PyLossless :octicon:`pulse` +=========================== +.. rst-class:: lead + EEG Processing Pipeline that is non-destructive, automated, and built on Python. -.. grid:: - - .. grid-item-card:: +.. container:: buttons - |:mechanical_arm:| Automated - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Automatic Processing Pipeline - cleans your EEG data. |:broom:| + `Docs `_ + `GitHub `_ - .. grid-item-card:: +.. grid:: 1 1 2 3 + :gutter: 2 + :padding: 0 + :class-row: surface - |:snake:| Built on Python - ^^^^^^^^^^^^^^^^^^^^^^^^^ - Ported from MATLAB for easier - use and access! + .. grid-item-card:: :octicon:`zap` Automated - .. grid-item-card:: + Fast, Open-source, and built on python. - |:recycle:| Non-destructive - ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Keeps your EEG continuous, so - you can epoch your data however - and whenever you want to. + .. grid-item-card:: :octicon:`pin` Non-destructive -.. grid:: + Keeps EEG continuous, noting bad channels, times, and independent components + so you can reject them and epoch your data however and whenever you want to. - .. grid-item-card:: + .. grid-item-card:: :octicon:`telescope-fill` Streamlined Review - |:pencil:| Artifacts are Noted - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Bad channels, times, and - components are stored as - ``Annotations`` in your - raw data. + Web dashboard built with helps you review the output and make + informed decisions about your data. - .. grid-item-card:: - - |:woman_technologist:| Streamlined Review - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Web dashboard built with Plotly/Dash helps - you Review the pipeline output and make - informed decisions about your data - .. image:: https://raw.githubusercontent.com/scott-huberty/wip_pipeline-figures/main/dashboard.png :alt: pylossless-qc-dashboard-screenshot - :align: center \ No newline at end of file + :align: center + + +.. toctree:: + :maxdepth: 1 + :hidden: + + install + auto_examples/index.rst + API/API_index + contributing + Paper \ No newline at end of file diff --git a/examples/README.txt b/examples/README.txt index 762332d..2e4fcde 100644 --- a/examples/README.txt +++ b/examples/README.txt @@ -1,2 +1,2 @@ -PyLossless Tutorials -^^^^^^^^^^^^^^^^^^^^ \ No newline at end of file +Tutorials +^^^^^^^^^ \ No newline at end of file diff --git a/pylossless/_logging.py b/pylossless/_logging.py index c44a98b..0086519 100644 --- a/pylossless/_logging.py +++ b/pylossless/_logging.py @@ -43,7 +43,7 @@ def wrapper(*args, message=None, **kwargs): logger.info(f"LOSSLESS: Skipping {this_step}") return elif verbose: - logger.info(f"LOSSLESS: 🚩 {this_step}.") + logger.info(f"LOSSLESS: 👇 {this_step}.") result = func(*args, **kwargs) end_time = time.time() dur = f"{end_time - start_time:.2f}" diff --git a/pylossless/conftest.py b/pylossless/conftest.py index a6ae964..d2080eb 100644 --- a/pylossless/conftest.py +++ b/pylossless/conftest.py @@ -13,6 +13,9 @@ import pytest +# TODO: Remove this once mne 1.7 is released +pytest.mark.filterwarnings("ignore:The current default of copy") + @pytest.fixture(scope="session") def pipeline_fixture(): diff --git a/pylossless/flagging.py b/pylossless/flagging.py index 04e1df1..02086dc 100644 --- a/pylossless/flagging.py +++ b/pylossless/flagging.py @@ -197,9 +197,8 @@ def add_flag_cat(self, kind, bad_epoch_inds, epochs): def load_from_raw(self, raw): """Load pylossless annotations from raw object.""" sfreq = raw.info["sfreq"] - lossless_descriptions = ["bad_noisy", "bad_uncorrelated", "bad_noisy_ICs"] for annot in raw.annotations: - if annot["description"] in lossless_descriptions: + if annot["description"].upper().startswith("BAD_LL"): ind_onset = int(np.round(annot["onset"] * sfreq)) ind_dur = int(np.round(annot["duration"] * sfreq)) inds = np.arange(ind_onset, ind_onset + ind_dur) diff --git a/pylossless/pipeline.py b/pylossless/pipeline.py index 8be3c93..c672a9a 100644 --- a/pylossless/pipeline.py +++ b/pylossless/pipeline.py @@ -31,6 +31,7 @@ from .config import Config from .flagging import FlaggedChs, FlaggedEpochs, FlaggedICs from ._logging import lossless_logger, lossless_time +from .utils import _report_flagged_epochs from .utils.html import _get_ics, _sum_flagged_times, _create_html_details @@ -476,9 +477,9 @@ def _repr_html_(self): channel_noise = _get_ics(df, "channel_noise") lossless_flags = [ - "bad_noisy", - "bad_uncorrelated", - "bad_noisy_ICs", + "BAD_LL_noisy", + "BAD_LL_uncorrelated", + "BAD_LL_noisy_ICs", ] flagged_times = _sum_flagged_times(self.raw, lossless_flags) @@ -579,13 +580,14 @@ def add_pylossless_annotations(self, inds, event_type, epochs): """ # Concatenate epoched data back to continuous data t_onset = epochs.events[inds, 0] / epochs.info["sfreq"] + desc = f"BAD_LL_{event_type}" df = pd.DataFrame(t_onset, columns=["onset"]) # We exclude the last sample from the duration because # if the annot lasts the whole duration of the epoch # it's end will coincide with the first sample of the # next epoch, causing it to erroneously be rejected. df["duration"] = 1 / epochs.info["sfreq"] * len(epochs.times[:-1]) - df["description"] = f"bad_{event_type}" + df["description"] = desc # Merge close onsets to prevent a bunch of 1-second annotations of the same name # find onsets close enough to be considered the same @@ -604,6 +606,7 @@ def add_pylossless_annotations(self, inds, event_type, epochs): orig_time=self.raw.annotations.orig_time, ) self.raw.set_annotations(self.raw.annotations + annotations) + _report_flagged_epochs(self.raw, desc) def get_events(self): """Make an MNE events array of fixed length events.""" @@ -865,7 +868,6 @@ def flag_noisy_epochs(self): bad_epoch_inds = _detect_outliers( data_sd, flag_dim="epoch", init_dir="pos", **config_epoch ) - logger.info(f"📋 LOSSLESS: Noisy epochs: {bad_epoch_inds}") self.flags["epoch"].add_flag_cat("noisy", bad_epoch_inds, epochs) def get_n_nbr(self): @@ -976,7 +978,6 @@ def flag_uncorrelated_epochs(self): init_dir="neg", **self.config["uncorrelated_epochs"], ) - logger.info(f"📋 LOSSLESS: Uncorrelated epochs: {bad_epoch_inds}") self.flags["epoch"].add_flag_cat("uncorrelated", bad_epoch_inds, epochs) @lossless_logger @@ -1175,7 +1176,7 @@ def _run(self): self.flag_noisy_ics(message="Flagging time periods with noisy IC's.") # 12. TODO: integrate labels from IClabels to self.flags["ic"] - self.run_ica("run2", message="Running Final ICA.") + self.run_ica("run2", message="Running Final ICA and ICLabel.") def run_dataset(self, paths): """Run a full dataset. diff --git a/pylossless/utils/__init__.py b/pylossless/utils/__init__.py index 4f3769d..33d54c7 100644 --- a/pylossless/utils/__init__.py +++ b/pylossless/utils/__init__.py @@ -1,2 +1,2 @@ -from ._utils import _icalabel_to_data_frame +from ._utils import _icalabel_to_data_frame, _report_flagged_epochs from .html import _get_ics, _sum_flagged_times, _create_html_details diff --git a/pylossless/utils/_utils.py b/pylossless/utils/_utils.py index 2cf5f4d..014d57b 100644 --- a/pylossless/utils/_utils.py +++ b/pylossless/utils/_utils.py @@ -7,7 +7,11 @@ """Utility Functions for running the Lossless Pipeline.""" +import numpy as np import pandas as pd +from mne.utils import logger + +from .html import _sum_flagged_times def _icalabel_to_data_frame(ica): @@ -26,3 +30,11 @@ def _icalabel_to_data_frame(ica): confidence=ica.labels_scores_.max(1), ) ) + + +def _report_flagged_epochs(raw, flag): + times = _sum_flagged_times(raw, flag)[flag] + if not times: + times = 0 + else: + logger.info(f"📋 LOSSLESS: {np.round(times, 2)} second(s) flagged as {flag}") diff --git a/pylossless/utils/html.py b/pylossless/utils/html.py index 1157ed8..490fff1 100644 --- a/pylossless/utils/html.py +++ b/pylossless/utils/html.py @@ -10,6 +10,8 @@ def _get_ics(df, ic_type): def _sum_flagged_times(raw, flags): """Sum the total time flagged for various flags like noisy etc.""" flag_dict = {} + if isinstance(flags, str): + flags = [flags] for flag in flags: flag_dict[flag] = [] if raw: diff --git a/requirements.txt b/requirements.txt index 7037537..169940b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,6 @@ mne_bids pandas xarray scipy>=1.2.1 -mne_icalabel +mne_icalabel>=0.5.0 pyyaml scikit-learn \ No newline at end of file