From 6361d7b041f0d1633bd273bde61ad6a218dab98e Mon Sep 17 00:00:00 2001 From: tensionhead Date: Mon, 21 Mar 2022 17:19:26 +0100 Subject: [PATCH 01/38] CHG: Allow for different spectral normalizations - added a `mode` parameter to _norm_spec On branch FT-normalization Changes to be committed: modified: syncopy/specest/_norm_spec.py --- syncopy/specest/_norm_spec.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/syncopy/specest/_norm_spec.py b/syncopy/specest/_norm_spec.py index 036bea17c..9966a7c5f 100644 --- a/syncopy/specest/_norm_spec.py +++ b/syncopy/specest/_norm_spec.py @@ -6,15 +6,19 @@ import numpy as np -def _norm_spec(ftr, nSamples, fs): +def _norm_spec(ftr, nSamples, fs, mode='amplitude'): """ Normalizes the complex Fourier transform to - power spectral density units. + power spectral density of amplitude squaredunits. """ # frequency bins - delta_f = fs / nSamples + if mode == 'density': + delta_f = fs / nSamples + elif mode == 'amplitude': + delta_f = 1 + ftr *= np.sqrt(2) / (nSamples * np.sqrt(delta_f)) return ftr From 087992e1c8fa126e5d2cad2427464883e05d518d Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 23 Mar 2022 14:42:41 +0100 Subject: [PATCH 02/38] CHG: Try dimensionless bins for spectral normalization On branch FT-normalization Changes to be committed: modified: syncopy/specest/_norm_spec.py modified: syncopy/specest/mtmfft.py --- syncopy/specest/_norm_spec.py | 6 +++--- syncopy/specest/mtmfft.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/syncopy/specest/_norm_spec.py b/syncopy/specest/_norm_spec.py index 9966a7c5f..70f3afbf0 100644 --- a/syncopy/specest/_norm_spec.py +++ b/syncopy/specest/_norm_spec.py @@ -6,17 +6,17 @@ import numpy as np -def _norm_spec(ftr, nSamples, fs, mode='amplitude'): +def _norm_spec(ftr, nSamples, fs, mode='bins'): """ Normalizes the complex Fourier transform to - power spectral density of amplitude squaredunits. + power spectral density or dimensionless bin units. """ # frequency bins if mode == 'density': delta_f = fs / nSamples - elif mode == 'amplitude': + elif mode == 'bins': delta_f = 1 ftr *= np.sqrt(2) / (nSamples * np.sqrt(delta_f)) diff --git a/syncopy/specest/mtmfft.py b/syncopy/specest/mtmfft.py index 752835bf7..8ab5c8ca7 100644 --- a/syncopy/specest/mtmfft.py +++ b/syncopy/specest/mtmfft.py @@ -102,6 +102,8 @@ def mtmfft(data_arr, if demean_taper: win -= win.mean(axis=0) ftr[taperIdx] = np.fft.rfft(win, n=nSamples, axis=0) - ftr[taperIdx] = _norm_spec(ftr[taperIdx], nSamples, samplerate) + # FT uses original length + ftr[taperIdx] = _norm_spec(ftr[taperIdx], signal_length, samplerate) + # ftr[taperIdx] = _norm_spec(ftr[taperIdx], nSamples, samplerate) return ftr, freqs From 627c59bf818288f85b0bbd5ceee54bcf1a4f033e Mon Sep 17 00:00:00 2001 From: KatharineShapcott <65502584+KatharineShapcott@users.noreply.github.com> Date: Tue, 19 Apr 2022 10:46:47 +0200 Subject: [PATCH 03/38] FIX: trialtime property Removed unique as that made the _get_time function incorrect. Code is now faster and identical to .time property in continuous_data. --- syncopy/datatype/discrete_data.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/syncopy/datatype/discrete_data.py b/syncopy/datatype/discrete_data.py index 21da49a31..b73a56258 100644 --- a/syncopy/datatype/discrete_data.py +++ b/syncopy/datatype/discrete_data.py @@ -182,9 +182,8 @@ def trials(self): def trialtime(self): """list(:class:`numpy.ndarray`): trigger-relative sample times in s""" if self.samplerate is not None and self.sampleinfo is not None: - return [np.array([(t + self._t0[tk]) / self.samplerate \ - for t in range(0, int(self.sampleinfo[tk, 1] - self.sampleinfo[tk, 0]))]) \ - for tk in np.unique(self.trialid)] + return [(np.arange(0, stop - start) + self._t0[tk]) / self.samplerate \ + for tk, (start, stop) in enumerate(self.sampleinfo)] # Helper function that grabs a single trial def _get_trial(self, trialno): From 69306016cbe049ad96ccbd7fbefcb1ad24511b9f Mon Sep 17 00:00:00 2001 From: tensionhead Date: Thu, 21 Apr 2022 12:37:15 +0200 Subject: [PATCH 04/38] CHG: Normalize after padding - now both taper and spectrum get normalized with the length after tapering - for hann and boxcar (no taper) the results between FT and Syncopy now match Changes to be committed: modified: syncopy/specest/_norm_spec.py modified: syncopy/specest/mtmfft.py --- syncopy/specest/_norm_spec.py | 7 +++++-- syncopy/specest/mtmfft.py | 6 ++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/syncopy/specest/_norm_spec.py b/syncopy/specest/_norm_spec.py index 70f3afbf0..ea46173f7 100644 --- a/syncopy/specest/_norm_spec.py +++ b/syncopy/specest/_norm_spec.py @@ -34,9 +34,12 @@ def _norm_taper(taper, windows, nSamples): if taper == 'dpss': windows *= np.sqrt(nSamples) + # only for padding + if taper == 'boxcar': + windows *= np.sqrt(nSamples / windows.sum()) # weird 3 point normalization, - # checks out exactly for 'hann' though - elif taper != 'boxcar': + # checks out (almost) exactly for 'hann' though + else: windows *= np.sqrt(4 / 3) * np.sqrt(nSamples / windows.sum()) return windows diff --git a/syncopy/specest/mtmfft.py b/syncopy/specest/mtmfft.py index 8ab5c8ca7..11b6d5914 100644 --- a/syncopy/specest/mtmfft.py +++ b/syncopy/specest/mtmfft.py @@ -89,7 +89,7 @@ def mtmfft(data_arr, # only really 2d if taper='dpss' with Kmax > 1 # here we take the actual signal lengths! windows = np.atleast_2d(taper_func(signal_length, **taper_opt)) - # normalize window + # normalize window with length after padding windows = _norm_taper(taper, windows, nSamples) # Fourier transforms (nTapers x nFreq x nChannels) @@ -102,8 +102,6 @@ def mtmfft(data_arr, if demean_taper: win -= win.mean(axis=0) ftr[taperIdx] = np.fft.rfft(win, n=nSamples, axis=0) - # FT uses original length - ftr[taperIdx] = _norm_spec(ftr[taperIdx], signal_length, samplerate) - # ftr[taperIdx] = _norm_spec(ftr[taperIdx], nSamples, samplerate) + ftr[taperIdx] = _norm_spec(ftr[taperIdx], nSamples, samplerate) return ftr, freqs From 5478d384a785d96764314411a532608d6e9bb669 Mon Sep 17 00:00:00 2001 From: KatharineShapcott <65502584+KatharineShapcott@users.noreply.github.com> Date: Thu, 21 Apr 2022 16:43:38 +0200 Subject: [PATCH 05/38] FIX: EventData can have flexible nCol --- syncopy/datatype/base_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 042fd6f0f..4b7c83c39 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -379,7 +379,7 @@ def _set_dataset_property_with_list(self, inData, propertyName, ndim): if self.__class__.__name__ == "SpikeData": nCol = 3 else: # EventData - nCol = 2 + nCol = inData[0].shape[1] if any(val.shape[1] != nCol for val in inData): lgl = "NumPy 2d-arrays with 3 columns" act = "NumPy arrays of different shape" From 5511acefd3228c8ee46c47f2363de7d5434b0037 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 26 Apr 2022 20:45:39 +0200 Subject: [PATCH 06/38] FIX: Improve specest tests - decrease testing data-set size used in test_specest.py to reduce memory consumption of tests (when running `pytest --full`); closes #268 - fixed testing setup on Windows: do not attempt to query open file ulimit - include `psutil` in conda-dependencies to not have `pip` trying to build it on ppc64 - todo: fix a bug that causes equidistant `toi` arrays with large spacing (> 1s) to trigger shape mismatches On branch dev Changes to be committed: modified: syncopy/tests/conftest.py modified: syncopy/tests/test_specest.py modified: tox.ini --- syncopy/tests/conftest.py | 15 ++++----- syncopy/tests/test_specest.py | 63 ++++++++++++++++++++++------------- tox.ini | 1 + 3 files changed, 47 insertions(+), 32 deletions(-) diff --git a/syncopy/tests/conftest.py b/syncopy/tests/conftest.py index cde8fc8b0..3ea08993b 100644 --- a/syncopy/tests/conftest.py +++ b/syncopy/tests/conftest.py @@ -4,10 +4,8 @@ # # Builtin/3rd party package imports -import os -import importlib +import sys import pytest -import syncopy from syncopy import __acme__ import syncopy.tests.test_packagesetup as setupTestModule @@ -17,13 +15,14 @@ # skipped anyway) if __acme__: import dask.distributed as dd - import resource from acme.dask_helpers import esi_cluster_setup from syncopy.tests.misc import is_slurm_node - if max(resource.getrlimit(resource.RLIMIT_NOFILE)) < 1024: - msg = "Not enough open file descriptors allowed. Consider increasing " +\ - "the limit using, e.g., `ulimit -Sn 1024`" - raise ValueError(msg) + if sys.platform != "win32": + import resource + if max(resource.getrlimit(resource.RLIMIT_NOFILE)) < 1024: + msg = "Not enough open file descriptors allowed. Consider increasing " +\ + "the limit using, e.g., `ulimit -Sn 1024`" + raise ValueError(msg) if is_slurm_node(): cluster = esi_cluster_setup(partition="8GB", n_jobs=10, timeout=360, interactive=False, diff --git a/syncopy/tests/test_specest.py b/syncopy/tests/test_specest.py index 696d626d8..5278509ea 100644 --- a/syncopy/tests/test_specest.py +++ b/syncopy/tests/test_specest.py @@ -46,8 +46,10 @@ def _make_tf_signal(nChannels, nTrials, seed, fadeIn=None, fadeOut=None): numType = "float32" modPeriods = [0.125, 0.0625] rng = np.random.default_rng(seed) - tStart = -29.5 - tStop = 70.5 + tStart = -2.95 # FIXME + tStop = 7.05 + # tStart = -29.5 + # tStop = 70.5 t0 = -np.abs(tStart * fs).astype(np.intp) time = (np.arange(0, (tStop - tStart) * fs, dtype=numType) + tStart * fs) / fs N = time.size @@ -60,7 +62,7 @@ def _make_tf_signal(nChannels, nTrials, seed, fadeIn=None, fadeOut=None): if fadeOut is None: fadeOut = tStop fadeIn = np.arange(0, (fadeIn - tStart) * fs, dtype=np.intp) - fadeOut = np.arange((fadeOut - tStart) * fs, 100 * fs, dtype=np.intp) + fadeOut = np.arange((fadeOut - tStart) * fs, 10 * fs, dtype=np.intp) sigmoid = lambda x: 1 / (1 + np.exp(-x)) fader[fadeIn] = sigmoid(np.linspace(-2 * np.pi, 2 * np.pi, fadeIn.size)) fader[fadeOut] = sigmoid(-np.linspace(-2 * np.pi, 2 * np.pi, fadeOut.size)) @@ -486,7 +488,8 @@ class TestMTMConvol(): "channel": ["channel" + str(i) for i in range(2, 6)][::-1]}, {"trials": [0, 2], "channel": range(0, nChan2), - "toilim": [-20, 60.8]}] + "toilim": [-2, 6.8]}] + # "toilim": [-20, 60.8]}] FIXME # Helper function that reduces dataselections (keep `None` selection no matter what) def test_tf_cut_selections(self, fulltests): @@ -498,7 +501,7 @@ def test_tf_output(self, fulltests): cfg = get_defaults(freqanalysis) cfg.method = "mtmconvol" cfg.taper = "hann" - cfg.toi = np.linspace(-20, 60, 10) + cfg.toi = np.linspace(-2, 6, 10) cfg.t_ftimwin = 1.0 outputDict = {"fourier" : "complex", "abs" : "float", "pow" : "float"} @@ -512,7 +515,7 @@ def test_tf_output(self, fulltests): else: # randomly pick from 'fourier', 'abs' and 'pow' and work w/smaller signal cfg.select = {"trials" : 0, "channel" : 1} cfg.output = random.choice(list(outputDict.keys())) - cfg.toi = np.linspace(-20, 60, 5) + cfg.toi = np.linspace(-2, 6, 5) tfSpec = freqanalysis(cfg, _make_tf_signal(2, 2, self.seed, fadeIn=self.fadeIn, fadeOut=self.fadeOut)[0]) assert outputDict[cfg.output] in tfSpec.data.dtype.name @@ -665,9 +668,12 @@ def test_tf_toi(self): # arrays containing the onset, purely pre-onset, purely after onset and # non-unit spacing toiVals = [0.9, 0.75] - toiArrs = [np.arange(-10, 15.1), - np.arange(-15, -10, 1/self.tfData.samplerate), - np.arange(1, 20, 2)] + toiArrs = [np.arange(-2,7), + np.arange(-1, 6, 1/self.tfData.samplerate), + np.arange(1, 6, 2)] + # toiArrs = [np.arange(-10, 15.1), + # np.arange(-15, -10, 1/self.tfData.samplerate), + # np.arange(1, 20, 2)] winSizes = [0.5, 1.0] # Combine `toi`-testing w/in-place data-pre-selection @@ -700,7 +706,7 @@ def test_tf_toi(self): assert tfSpec.samplerate == 1/(toi[1] - toi[0]) # Unevenly sampled array: timing currently in lala-land, but sizes must match - cfg.toi = [-5, 3, 10] + cfg.toi = [-1, 2, 6] tfSpec = freqanalysis(cfg, self.tfData) assert tfSpec.time[0].size == len(cfg.toi) @@ -910,8 +916,8 @@ class TestWavelet(): nChannels = 4 nTrials = 3 seed = 151120 - fadeIn = -9.5 - fadeOut = 50.5 + fadeIn = -1.5 + fadeOut = 5.5 tfData, modulators, even, odd, fader = _make_tf_signal(nChannels, nTrials, seed, fadeIn=fadeIn, fadeOut=fadeOut) @@ -921,7 +927,7 @@ class TestWavelet(): "channel": ["channel" + str(i) for i in range(2, 4)][::-1]}, {"trials": [0, 2], "channel": range(0, int(nChannels / 2)), - "toilim": [-20, 60.8]}] + "toilim": [-2, 6.8]}] # Helper function that reduces dataselections (keep `None` selection no matter what) def test_wav_cut_selections(self, fulltests): @@ -939,6 +945,9 @@ def test_wav_solution(self): cfg.width = 1 cfg.output = "pow" + # import pdb; pdb.set_trace() + + # Set up index tuple for slicing computed TF spectra and collect values # of expected frequency peaks (for validation of `foi`/`foilim` selections below) chanIdx = SpectralData._defaultDimord.index("channel") @@ -975,9 +984,9 @@ def test_wav_solution(self): cfg.foilim = None # Ensure TF objects contain expected/requested frequencies - assert 0.02 > tfSpec.freq.min() > 0 + assert 0.2 > tfSpec.freq.min() > 0 assert tfSpec.freq.max() == (self.tfData.samplerate / 2) - assert tfSpec.freq.size > 60 + assert tfSpec.freq.size > 40 assert np.allclose(tfSpecFoi.freq, maxFreqs) assert np.allclose(tfSpecFoiLim.freq, foilimFreqs) @@ -1053,9 +1062,12 @@ def test_wav_toi(self): # Test time-point arrays comprising onset, purely pre-onset, purely after # onset and non-unit spacing - toiArrs = [np.arange(-10, 15.1), - np.arange(-15, -10, 1/self.tfData.samplerate), - np.arange(1, 20, 2)] + toiArrs = [np.arange(-2,7), + np.arange(-1, 6, 1/self.tfData.samplerate), + np.arange(1, 6, 2)] + # toiArrs = [np.arange(-10, 15.1), + # np.arange(-15, -10, 1/self.tfData.samplerate), + # np.arange(1, 20, 2)] # Combine `toi`-testing w/in-place data-pre-selection for select in self.dataSelections: @@ -1217,7 +1229,7 @@ class TestSuperlet(): "channel": ["channel" + str(i) for i in range(2, 4)][::-1]}, {"trials": [0, 2], "channel": range(0, int(nChannels / 2)), - "toilim": [-20, 60.8]}] + "toilim": [-2, 6.8]}] # Helper function that reduces dataselections (keep `None` selection no matter what) def test_slet_cut_selections(self, fulltests): @@ -1273,9 +1285,9 @@ def test_slet_solution(self, fulltests): tfSpec = freqanalysis(cfg, self.tfData) assert 0.02 > tfSpec.freq.min() > 0 assert tfSpec.freq.max() == (self.tfData.samplerate / 2) - assert tfSpec.freq.size > 60 + assert tfSpec.freq.size > 50 - for tk, trlArr in enumerate(tfSpecFoi.trials): + for tk in tfSpecFoi.trials: # Get reference trial-number in input object trlNo = tk @@ -1336,9 +1348,12 @@ def test_slet_toi(self, fulltests): # Test time-point arrays comprising onset, purely pre-onset, purely after # onset and non-unit spacing - toiArrs = [np.arange(-10, 15.1), - np.arange(-15, -10, 1/self.tfData.samplerate), - np.arange(1, 20, 2)] + toiArrs = [np.arange(-2,7), + np.arange(-1, 6, 1/self.tfData.samplerate), + np.arange(1, 6, 2)] + # toiArrs = [np.arange(-10, 15.1), + # np.arange(-15, -10, 1/self.tfData.samplerate), + # np.arange(1, 20, 2)] # Just pick one `toi` at random for quickly running tests if not fulltests: diff --git a/tox.ini b/tox.ini index a7bce3d9a..3be7ea46e 100644 --- a/tox.ini +++ b/tox.ini @@ -16,6 +16,7 @@ conda_deps= scipy15: scipy >= 1.5 acme: esi-acme matplotlib >= 3.3, < 3.5 + psutil conda_channels= defaults conda-forge From cbd875984da4f64ebd11d82df1cf7c5086dfe499 Mon Sep 17 00:00:00 2001 From: Katharine Shapcott Date: Thu, 28 Apr 2022 16:02:37 +0200 Subject: [PATCH 07/38] CHG: DiscreteData time and trialtime properties time is a list of trial relative times in seconds. trialtime is a vector of trial relative times in seconds, or a nan when outside the trial. selectdata option toi is no longer supported. --- CITATION.cff | 4 +- syncopy/datatype/base_data.py | 5 +++ syncopy/datatype/discrete_data.py | 62 +++++++++++-------------------- 3 files changed, 28 insertions(+), 43 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index d04a5701d..3c818c77c 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -37,5 +37,5 @@ keywords: - spectral-methods - brain repository-code: https://github.com/esi-neuroscience/syncopy -version: 0.1b3.dev287 -date-released: '2022-01-19' +version: 0.3.dev187 +date-released: '2022-04-13' diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 4b7c83c39..fb483f7e0 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -1500,6 +1500,11 @@ def __init__(self, data, select): if self._dataClass == "CrossSpectralData": self._dimProps.remove("channel") + # DiscreteData cannot be selected by toi + if self._dataClass in ["EventData","SpikeData"]: + if select.get("toi") is not None: + raise(ValueError("toi cannot be used to select %s"%(self._dataClass))) + # Assign defaults (trials are not a "real" property, handle it separately, # same goes for `trialdefinition`) self._trials = None diff --git a/syncopy/datatype/discrete_data.py b/syncopy/datatype/discrete_data.py index b73a56258..c107de480 100644 --- a/syncopy/datatype/discrete_data.py +++ b/syncopy/datatype/discrete_data.py @@ -145,6 +145,13 @@ def samplerate(self, sr): raise exc self._samplerate = sr + @property + def time(self): + """list(float): trigger-relative time of each event """ + if self.samplerate is not None and self.sampleinfo is not None: + return [(trl[:,self.dimord.index("sample")] - self.sampleinfo[tk,0] + self._t0[tk]) / self.samplerate \ + for tk, trl in enumerate(self.trials)] + @property def trialid(self): """:class:`numpy.ndarray` of trial id associated with the sample""" @@ -182,8 +189,9 @@ def trials(self): def trialtime(self): """list(:class:`numpy.ndarray`): trigger-relative sample times in s""" if self.samplerate is not None and self.sampleinfo is not None: - return [(np.arange(0, stop - start) + self._t0[tk]) / self.samplerate \ - for tk, (start, stop) in enumerate(self.sampleinfo)] + sample0 = self.sampleinfo[:,0] + self._t0[:] + sample0 = np.append(sample0, np.nan)[self.trialid] + return (self.data[:,self.dimord.index("sample")] - sample0)/self.samplerate # Helper function that grabs a single trial def _get_trial(self, trialno): @@ -229,7 +237,7 @@ def _preview_trial(self, trialno): return FauxTrial(shp, tuple(idx), self.data.dtype, self.dimord) # Helper function that extracts by-trial timing-related indices - def _get_time(self, trials, toi=None, toilim=None): + def _get_time(self, trials, toilim=None, toi=None): """ Get relative by-trial indices of time-selections @@ -237,16 +245,16 @@ def _get_time(self, trials, toi=None, toilim=None): ---------- trials : list List of trial-indices to perform selection on - toi : None or list - Time-points to be selected (in seconds) on a by-trial scale. toilim : None or list Time-window to be selected (in seconds) on a by-trial scale + toi : None + Should always be none for DiscreteData Returns ------- timing : list of lists List of by-trial sample-indices corresponding to provided - time-selection. If both `toi` and `toilim` are `None`, `timing` + time-selection. If `toilim` is `None`, `timing` is a list of universal (i.e., ``slice(None)``) selectors. Notes @@ -265,41 +273,13 @@ def _get_time(self, trials, toi=None, toilim=None): if toilim is not None: allTrials = self.trialtime for trlno in trials: - thisTrial = self.data[self.trialid == trlno, self.dimord.index("sample")] - trlSample = np.arange(*self.sampleinfo[trlno, :]) - trlTime = allTrials[trlno] - minSample = trlSample[np.where(trlTime >= toilim[0])[0][0]] - maxSample = trlSample[np.where(trlTime <= toilim[1])[0][-1]] - selSample, _ = best_match(trlSample, [minSample, maxSample], span=True) - idxList = [] - for smp in selSample: - idxList += list(np.where(thisTrial == smp)[0]) - if len(idxList) > 1: - sampSteps = np.diff(idxList) - if sampSteps.min() == sampSteps.max() == 1: - idxList = slice(idxList[0], idxList[-1] + 1, 1) - timing.append(idxList) - - elif toi is not None: - allTrials = self.trialtime - for trlno in trials: - thisTrial = self.data[self.trialid == trlno, self.dimord.index("sample")] - trlSample = np.arange(*self.sampleinfo[trlno, :]) - trlTime = allTrials[trlno] - _, selSample = best_match(trlTime, toi) - for k, idx in enumerate(selSample): - if np.abs(trlTime[idx - 1] - toi[k]) < np.abs(trlTime[idx] - toi[k]): - selSample[k] = trlSample[idx -1] - else: - selSample[k] = trlSample[idx] - idxList = [] - for smp in selSample: - idxList += list(np.where(thisTrial == smp)[0]) - if len(idxList) > 1: - sampSteps = np.diff(idxList) - if sampSteps.min() == sampSteps.max() == 1: - idxList = slice(idxList[0], idxList[-1] + 1, 1) - timing.append(idxList) + trlTime = allTrials[self.trialid == trlno] + _, selTime = best_match(trlTime, toilim, span=True) + selTime = selTime.tolist() + if len(selTime) > 1: + timing.append(slice(selTime[0], selTime[-1] + 1, 1)) + else: + timing.append(selTime) else: timing = [slice(None)] * len(trials) From 2270bb1accb49815ead4fb270edc67784b1a154b Mon Sep 17 00:00:00 2001 From: Katharine Shapcott Date: Thu, 28 Apr 2022 16:06:29 +0200 Subject: [PATCH 08/38] FIX: Removed toi from discretedata tests --- syncopy/tests/test_discretedata.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/syncopy/tests/test_discretedata.py b/syncopy/tests/test_discretedata.py index 1cadec4c3..106f3af97 100644 --- a/syncopy/tests/test_discretedata.py +++ b/syncopy/tests/test_discretedata.py @@ -171,10 +171,6 @@ def test_dataselection(self, fulltests): range(5, 8), # narrow range slice(-5, None) # negative-start slice ] - toiSelections = [ - "all", # non-type-conform string - [-0.2, 0.6, 0.9, 1.1, 1.3, 1.6, 1.8, 2.2, 2.45, 3.] # unordered, inexact, repetions - ] toilimSelections = [ [0.5, 3.5], # regular range [1.0, np.inf] # unbounded from above @@ -185,8 +181,7 @@ def test_dataselection(self, fulltests): range(1, 4), # narrow range slice(-2, None) # negative-start slice ] - timeSelections = list(zip(["toi"] * len(toiSelections), toiSelections)) \ - + list(zip(["toilim"] * len(toilimSelections), toilimSelections)) + timeSelections = list(zip(["toilim"] * len(toilimSelections), toilimSelections)) # Randomly pick one selection unless tests are run with `--full` if fulltests: @@ -513,16 +508,11 @@ def test_ed_dataselection(self, fulltests): range(0, 2), # narrow range slice(-2, None) # negative-start slice ] - toiSelections = [ - "all", # non-type-conform string - [-0.2, 0.6, 0.9, 1.1, 1.3, 1.6, 1.8, 2.2, 2.45, 3.] # unordered, inexact, repetions - ] toilimSelections = [ [0.5, 3.5], # regular range [0.0, np.inf] # unbounded from above ] - timeSelections = list(zip(["toi"] * len(toiSelections), toiSelections)) \ - + list(zip(["toilim"] * len(toilimSelections), toilimSelections)) + timeSelections = list(zip(["toilim"] * len(toilimSelections), toilimSelections)) # Randomly pick one selection unless tests are run with `--full` if fulltests: From c4187601a1685f4c78f2eb0db0d12aa3c30f75c3 Mon Sep 17 00:00:00 2001 From: Katharine Shapcott Date: Thu, 28 Apr 2022 18:48:06 +0200 Subject: [PATCH 09/38] FIX: Support selector toi input again --- syncopy/datatype/base_data.py | 5 ----- syncopy/datatype/discrete_data.py | 20 ++++++++++++++++---- syncopy/tests/test_discretedata.py | 14 ++++++++++++-- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index fb483f7e0..4b7c83c39 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -1500,11 +1500,6 @@ def __init__(self, data, select): if self._dataClass == "CrossSpectralData": self._dimProps.remove("channel") - # DiscreteData cannot be selected by toi - if self._dataClass in ["EventData","SpikeData"]: - if select.get("toi") is not None: - raise(ValueError("toi cannot be used to select %s"%(self._dataClass))) - # Assign defaults (trials are not a "real" property, handle it separately, # same goes for `trialdefinition`) self._trials = None diff --git a/syncopy/datatype/discrete_data.py b/syncopy/datatype/discrete_data.py index c107de480..d555c4914 100644 --- a/syncopy/datatype/discrete_data.py +++ b/syncopy/datatype/discrete_data.py @@ -237,7 +237,7 @@ def _preview_trial(self, trialno): return FauxTrial(shp, tuple(idx), self.data.dtype, self.dimord) # Helper function that extracts by-trial timing-related indices - def _get_time(self, trials, toilim=None, toi=None): + def _get_time(self, trials, toi=None, toilim=None): """ Get relative by-trial indices of time-selections @@ -245,16 +245,16 @@ def _get_time(self, trials, toilim=None, toi=None): ---------- trials : list List of trial-indices to perform selection on + toi : None or list + Time-points to be selected (in seconds) on a by-trial scale. toilim : None or list Time-window to be selected (in seconds) on a by-trial scale - toi : None - Should always be none for DiscreteData Returns ------- timing : list of lists List of by-trial sample-indices corresponding to provided - time-selection. If `toilim` is `None`, `timing` + time-selection. If both `toi` and `toilim` are `None`, `timing` is a list of universal (i.e., ``slice(None)``) selectors. Notes @@ -281,6 +281,18 @@ def _get_time(self, trials, toilim=None, toi=None): else: timing.append(selTime) + elif toi is not None: + allTrials = self.trialtime + for trlno in trials: + trlTime = allTrials[self.trialid == trlno] + _, selTime = best_match(trlTime, toi) + selTime = selTime.tolist() + if len(selTime) > 1: + timeSteps = np.diff(selTime) + if timeSteps.min() == timeSteps.max() == 1: + selTime = slice(selTime[0], selTime[-1] + 1, 1) + timing.append(selTime) + else: timing = [slice(None)] * len(trials) diff --git a/syncopy/tests/test_discretedata.py b/syncopy/tests/test_discretedata.py index 106f3af97..1cadec4c3 100644 --- a/syncopy/tests/test_discretedata.py +++ b/syncopy/tests/test_discretedata.py @@ -171,6 +171,10 @@ def test_dataselection(self, fulltests): range(5, 8), # narrow range slice(-5, None) # negative-start slice ] + toiSelections = [ + "all", # non-type-conform string + [-0.2, 0.6, 0.9, 1.1, 1.3, 1.6, 1.8, 2.2, 2.45, 3.] # unordered, inexact, repetions + ] toilimSelections = [ [0.5, 3.5], # regular range [1.0, np.inf] # unbounded from above @@ -181,7 +185,8 @@ def test_dataselection(self, fulltests): range(1, 4), # narrow range slice(-2, None) # negative-start slice ] - timeSelections = list(zip(["toilim"] * len(toilimSelections), toilimSelections)) + timeSelections = list(zip(["toi"] * len(toiSelections), toiSelections)) \ + + list(zip(["toilim"] * len(toilimSelections), toilimSelections)) # Randomly pick one selection unless tests are run with `--full` if fulltests: @@ -508,11 +513,16 @@ def test_ed_dataselection(self, fulltests): range(0, 2), # narrow range slice(-2, None) # negative-start slice ] + toiSelections = [ + "all", # non-type-conform string + [-0.2, 0.6, 0.9, 1.1, 1.3, 1.6, 1.8, 2.2, 2.45, 3.] # unordered, inexact, repetions + ] toilimSelections = [ [0.5, 3.5], # regular range [0.0, np.inf] # unbounded from above ] - timeSelections = list(zip(["toilim"] * len(toilimSelections), toilimSelections)) + timeSelections = list(zip(["toi"] * len(toiSelections), toiSelections)) \ + + list(zip(["toilim"] * len(toilimSelections), toilimSelections)) # Randomly pick one selection unless tests are run with `--full` if fulltests: From 02b0e9485be4b87500bd20de8cd08194f34079a2 Mon Sep 17 00:00:00 2001 From: Katharine Shapcott Date: Thu, 28 Apr 2022 19:44:11 +0200 Subject: [PATCH 10/38] FIX: EventData accepts extra dimord dimesion labels --- syncopy/datatype/discrete_data.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/syncopy/datatype/discrete_data.py b/syncopy/datatype/discrete_data.py index d555c4914..a8f1151b9 100644 --- a/syncopy/datatype/discrete_data.py +++ b/syncopy/datatype/discrete_data.py @@ -608,6 +608,15 @@ def __init__(self, :func:`syncopy.definetrial` """ + if dimord is not None: + # ensure that event data can have extra dimord columns + if data.shape[1] != len(self._defaultDimord): + for col in self._defaultDimord: + if col not in dimord: + base = "dimensional label {}" + lgl = base.format("'" + col + "'") + raise SPYValueError(legal=lgl, varname="dimord") + self._defaultDimord = dimord # Call parent initializer super().__init__(data=data, From 6839ac1692b203ae4523b6a6f609fe0dc94ba31d Mon Sep 17 00:00:00 2001 From: Katharine Shapcott Date: Fri, 29 Apr 2022 10:21:47 +0200 Subject: [PATCH 11/38] FIX: dimord length checked --- syncopy/datatype/discrete_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncopy/datatype/discrete_data.py b/syncopy/datatype/discrete_data.py index a8f1151b9..10219c4ba 100644 --- a/syncopy/datatype/discrete_data.py +++ b/syncopy/datatype/discrete_data.py @@ -610,7 +610,7 @@ def __init__(self, """ if dimord is not None: # ensure that event data can have extra dimord columns - if data.shape[1] != len(self._defaultDimord): + if len(dimord) != len(self._defaultDimord): for col in self._defaultDimord: if col not in dimord: base = "dimensional label {}" From 6a237da140b3f93efdcd37d42f6815a083374610 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 29 Apr 2022 15:40:56 +0200 Subject: [PATCH 12/38] FIX: Correctly process equidistant toi with large spacing - if `freqanalysis` is called with a `toi` array whose elements are equidistant but "sparse" (i.e., their spacing is large enough that the requested sliding windows do not overlap), treat this `toi` as if it was not equidistant, meaning call `mtmfft` at each time-point. On branch dev Changes to be committed: modified: syncopy/specest/freqanalysis.py --- syncopy/specest/freqanalysis.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index 0aff0183f..b1685e478 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -603,6 +603,11 @@ def freqanalysis(data, method='mtmfft', output='pow', padEnd = halfWin - offEnd padEnd = ((padEnd > 0) * padEnd).astype(np.intp) + # Treat equi-distant `toi` arrays with spacing large enough that windows + # do not overlap as if they were not equidistant + if tSteps.max() * data.samplerate > halfWin and equidistant: + equidistant = False + # Compute sample-indices (one slice/list per trial) from time-selections soi = [] if equidistant: From 3d1ab995a66410b3ced3c0313caacbab32dbb0c8 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 29 Apr 2022 15:41:46 +0200 Subject: [PATCH 13/38] CHG: Removed support for toi arrays for active in-place time-selections - if `freqanalysis` is invoked with a `toi` array as well as an input dataset that has an active in-place time-selection attached a `SPYValueError` is raised. On branch dev Changes to be committed: modified: syncopy/specest/freqanalysis.py --- syncopy/specest/freqanalysis.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index b1685e478..00566b0be 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -312,6 +312,10 @@ def freqanalysis(data, method='mtmfft', output='pow', # If only a subset of `data` is to be processed, make some necessary adjustments # of the sampleinfo and trial lengths if data.selection is not None: + # Refuse to go ahead with active time selection and provided `toi` on top` + if any(tsel != slice(None) for tsel in data.selection.time) and isinstance(toi, (np.ndarray, list)): + lgl = "no `toi` specification due to active in-place time-selection in input dataset" + raise SPYValueError(legal=lgl, varname="toi", actual=toi) sinfo = data.selection.trialdefinition[:, :2] trialList = data.selection.trials else: From 6fb89a7d20afab91acdcf93cbf666a8e15e79df1 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 29 Apr 2022 15:42:15 +0200 Subject: [PATCH 14/38] FIX: Modify specest tests - adjust specest tests to account for the changed logic in `freqanalysis` On branch dev Changes to be committed: modified: syncopy/tests/test_specest.py --- syncopy/tests/test_specest.py | 62 ++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/syncopy/tests/test_specest.py b/syncopy/tests/test_specest.py index 5278509ea..c50cdc5b6 100644 --- a/syncopy/tests/test_specest.py +++ b/syncopy/tests/test_specest.py @@ -508,6 +508,11 @@ def test_tf_output(self, fulltests): for select in self.dataSelections: if fulltests: cfg.select = select + if select is not None and "toilim" in cfg.select.keys(): + with pytest.raises(SPYValueError) as err: + freqanalysis(cfg, self.tfData) + assert "expected no `toi` specification due to active in-place time-selection" in str(err) + continue for key, value in outputDict.items(): cfg.output = key tfSpec = freqanalysis(cfg, self.tfData) @@ -698,17 +703,18 @@ def test_tf_toi(self): assert np.allclose(timeArr, tfSpec.time[0]) # Test window-centroids specified as time-point arrays - cfg.t_ftimwin = 0.05 - for toi in toiArrs: - cfg.toi = toi - tfSpec = freqanalysis(cfg, self.tfData) - assert np.allclose(cfg.toi, tfSpec.time[0]) - assert tfSpec.samplerate == 1/(toi[1] - toi[0]) + if select is not None and "toilim" not in select.keys(): + cfg.t_ftimwin = 0.05 + for toi in toiArrs: + cfg.toi = toi + tfSpec = freqanalysis(cfg, self.tfData) + assert np.allclose(cfg.toi, tfSpec.time[0]) + assert tfSpec.samplerate == 1/(toi[1] - toi[0]) - # Unevenly sampled array: timing currently in lala-land, but sizes must match - cfg.toi = [-1, 2, 6] - tfSpec = freqanalysis(cfg, self.tfData) - assert tfSpec.time[0].size == len(cfg.toi) + # Unevenly sampled array: timing currently in lala-land, but sizes must match + cfg.toi = [-1, 2, 6] + tfSpec = freqanalysis(cfg, self.tfData) + assert tfSpec.time[0].size == len(cfg.toi) # Test correct time-array assembly for ``toi = "all"`` (cut down data signifcantly # to not overflow memory here); same for ``toi = 1.0``` @@ -965,6 +971,7 @@ def test_wav_solution(self): timeArr = np.arange(self.tfData.time[0][0], self.tfData.time[0][-1]) if select: if "toilim" in select.keys(): + continue timeArr = np.arange(*select["toilim"]) timeStart = int(select['toilim'][0] * self.tfData.samplerate - self.tfData._t0[0]) timeStop = int(select['toilim'][1] * self.tfData.samplerate - self.tfData._t0[0]) @@ -1065,18 +1072,16 @@ def test_wav_toi(self): toiArrs = [np.arange(-2,7), np.arange(-1, 6, 1/self.tfData.samplerate), np.arange(1, 6, 2)] - # toiArrs = [np.arange(-10, 15.1), - # np.arange(-15, -10, 1/self.tfData.samplerate), - # np.arange(1, 20, 2)] # Combine `toi`-testing w/in-place data-pre-selection for select in self.dataSelections: - cfg.select = select - for toi in toiArrs: - cfg.toi = toi - tfSpec = freqanalysis(cfg, self.tfData) - assert np.allclose(cfg.toi, tfSpec.time[0]) - assert tfSpec.samplerate == 1/(toi[1] - toi[0]) + if select is not None and "toilim" not in select.keys(): + cfg.select = select + for toi in toiArrs: + cfg.toi = toi + tfSpec = freqanalysis(cfg, self.tfData) + assert np.allclose(cfg.toi, tfSpec.time[0]) + assert tfSpec.samplerate == 1/(toi[1] - toi[0]) # Test correct time-array assembly for ``toi = "all"`` (cut down data signifcantly # to not overflow memory here) @@ -1271,6 +1276,10 @@ def test_slet_solution(self, fulltests): timeSelection = np.where(self.fader == 1.0)[0] cfg.toi = timeArr + # Skip below tests if `toi` and an in-place time-selection clash + if select is not None and "toilim" in select.keys(): + continue + # Compute TF objects w\w/o`foi`/`foilim` cfg.select = select cfg.foi = maxFreqs @@ -1287,7 +1296,7 @@ def test_slet_solution(self, fulltests): assert tfSpec.freq.max() == (self.tfData.samplerate / 2) assert tfSpec.freq.size > 50 - for tk in tfSpecFoi.trials: + for tk, _ in enumerate(tfSpecFoi.trials): # Get reference trial-number in input object trlNo = tk @@ -1361,12 +1370,13 @@ def test_slet_toi(self, fulltests): # Combine `toi`-testing w/in-place data-pre-selection for select in self.dataSelections: - cfg.select = select - for toi in toiArrs: - cfg.toi = toi - tfSpec = freqanalysis(cfg, self.tfData) - assert np.allclose(cfg.toi, tfSpec.time[0]) - assert tfSpec.samplerate == 1/(toi[1] - toi[0]) + if select is not None and "toilim" not in select.keys(): + cfg.select = select + for toi in toiArrs: + cfg.toi = toi + tfSpec = freqanalysis(cfg, self.tfData) + assert np.allclose(cfg.toi, tfSpec.time[0]) + assert tfSpec.samplerate == 1/(toi[1] - toi[0]) # Test correct time-array assembly for ``toi = "all"`` (cut down data signifcantly # to not overflow memory here) From 05c0b00681be2ce4fa358a2671b7d3872349e10a Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 29 Apr 2022 15:42:32 +0200 Subject: [PATCH 15/38] FIX: Improve legibility of non-release versions - use `git describe --tags` instead of `--always` to get a version string relative to the latest (not necessarily annotated) tag On branch dev Changes to be committed: modified: syncopy/__init__.py --- syncopy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncopy/__init__.py b/syncopy/__init__.py index 11dca5a53..b8301c9de 100644 --- a/syncopy/__init__.py +++ b/syncopy/__init__.py @@ -16,7 +16,7 @@ try: __version__ = version("esi-syncopy") except PackageNotFoundError: - proc = subprocess.Popen("git describe --always", + proc = subprocess.Popen("git describe --tags", stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, shell=True) out, err = proc.communicate() From ad2c87b5057fe6011bde4440818d4008b90fbd1b Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 29 Apr 2022 15:46:32 +0200 Subject: [PATCH 16/38] CHG: Updated CHANGELOG - modified CHANGELOG for next release On branch dev Changes to be committed: modified: CHANGELOG.md --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9487ef444..a235475b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] +### NEW + +### CHANGED + +### REMOVED + +### DEPRECATED +- Removed support for calling `freqanalysis` with a `toi` array as well as an + input dataset that has an active in-place time-selection attached + +### FIXED +- Improved legibility of `spy.__version__` for non-release installations +- Correctly process equidistant `toi` arrays with large spacing in `freqanalysis` + ## [v0.21] - 2022-04-13 Feature update and bugfixes. From d8ce4228c37ba39f9b67e71d652a8596fab0f4e2 Mon Sep 17 00:00:00 2001 From: Katharine Shapcott Date: Tue, 3 May 2022 20:46:02 +0200 Subject: [PATCH 17/38] FIX: trialtime corrected and handling unsorted time --- syncopy/datatype/discrete_data.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/syncopy/datatype/discrete_data.py b/syncopy/datatype/discrete_data.py index 10219c4ba..f2c07b32f 100644 --- a/syncopy/datatype/discrete_data.py +++ b/syncopy/datatype/discrete_data.py @@ -189,7 +189,7 @@ def trials(self): def trialtime(self): """list(:class:`numpy.ndarray`): trigger-relative sample times in s""" if self.samplerate is not None and self.sampleinfo is not None: - sample0 = self.sampleinfo[:,0] + self._t0[:] + sample0 = self.sampleinfo[:,0] - self._t0[:] sample0 = np.append(sample0, np.nan)[self.trialid] return (self.data[:,self.dimord.index("sample")] - sample0)/self.samplerate @@ -276,7 +276,7 @@ def _get_time(self, trials, toi=None, toilim=None): trlTime = allTrials[self.trialid == trlno] _, selTime = best_match(trlTime, toilim, span=True) selTime = selTime.tolist() - if len(selTime) > 1: + if len(selTime) > 1 and np.diff(trlTime).min() > 0: timing.append(slice(selTime[0], selTime[-1] + 1, 1)) else: timing.append(selTime) @@ -285,8 +285,14 @@ def _get_time(self, trials, toi=None, toilim=None): allTrials = self.trialtime for trlno in trials: trlTime = allTrials[self.trialid == trlno] - _, selTime = best_match(trlTime, toi) - selTime = selTime.tolist() + _, arrayIdx = best_match(trlTime, toi) + # squash duplicate values then readd + _, xdi = np.unique(trlTime[arrayIdx], return_index=True) + arrayIdx = arrayIdx[np.sort(xdi)] + selTime = [] + for t in arrayIdx: + selTime += np.where(trlTime[t] == trlTime)[0].tolist() + # convert to slice if possible if len(selTime) > 1: timeSteps = np.diff(selTime) if timeSteps.min() == timeSteps.max() == 1: From 147376b8a4aa273395590553d40edd59bb57b59a Mon Sep 17 00:00:00 2001 From: Katharine Shapcott Date: Tue, 3 May 2022 20:46:46 +0200 Subject: [PATCH 18/38] FIX: test_discrete_toitoilim Now SpikeData and EventData contain all times for argmin functionality. --- syncopy/tests/test_selectdata.py | 138 +++++++++++++++++++++---------- 1 file changed, 95 insertions(+), 43 deletions(-) diff --git a/syncopy/tests/test_selectdata.py b/syncopy/tests/test_selectdata.py index a14ca1033..4baabfc4e 100644 --- a/syncopy/tests/test_selectdata.py +++ b/syncopy/tests/test_selectdata.py @@ -40,7 +40,7 @@ class TestSelector(): nTrials = 5 lenTrial = int(nSamples / nTrials) - 1 nFreqs = 15 - nSpikes = 50 + nSpikes = 100 samplerate = 2.0 data = {} trl = {} @@ -324,8 +324,8 @@ class TestSelector(): trl["SpikeData"] = trl["AnalogData"] # Use a triple-trigger pattern to simulate EventData w/non-uniform trials - data["EventData"] = np.vstack([np.arange(0, nSamples, 2), - np.zeros((int(nSamples / 2), ))]).T + data["EventData"] = np.vstack([np.arange(0, nSamples, 1), + np.zeros((int(nSamples), ))]).T data["EventData"][1::3, 1] = 1 data["EventData"][2::3, 1] = 2 trl["EventData"] = trl["AnalogData"] @@ -657,7 +657,7 @@ def test_discrete_toitoilim(self): [2.0, 0.5, 2.5], # unsorted list [1.0, 0.5, 0.5, 1.5], # repetition [0.5, 0.5, 1.0, 1.5], # preserve repetition, don't convert to slice - [0.5, 1.0, 1.5]), # sorted list (should be converted to slice-selection) + [0.5, 1.0, 1.5]), # sorted list "toilim": (None, # trivial "selection" of entire contents "all", # trivial "selection" of entire contents [0.5, 1.5], # regular range @@ -666,61 +666,113 @@ def test_discrete_toitoilim(self): [-np.inf, 1.0])} # unbounded from below # all trials have same time-scale for both `EventData` and `SpikeData`: take 1st one as reference - trlTime = list((np.arange(0, self.trl["SpikeData"][0, 1] - self.trl["SpikeData"][0, 0]) - + self.trl["SpikeData"][0, 2])/2 ) + # trlTime = (np.arange(0, self.trl["SpikeData"][0, 1] - self.trl["SpikeData"][0, 0]) + # + self.trl["SpikeData"][0, 2]) / self.samplerate # the below method of extracting spikes satisfying `toi`/`toilim` only works w/equidistant trials! for dclass in ["SpikeData", "EventData"]: discrete = getattr(spd, dclass)(data=self.data[dclass], trialdefinition=self.trl[dclass], samplerate=self.samplerate) + discrIdx = [slice(None)] * len(discrete.dimord) + #timeIdx = discrete.dimord.index("time") for tselect in ["toi", "toilim"]: for timeSel in selDict[tselect]: - if isinstance(timeSel, list): - smpIdx = [] - for tp in timeSel: - if np.isfinite(tp): - smpIdx.append(np.abs(np.array(trlTime) - tp).argmin()) - else: - smpIdx.append(tp) - result = [] + print({tselect: timeSel}) sel = Selector(discrete, {tselect: timeSel}).time - selected = selectdata(discrete, {tselect: timeSel}) - tk = 0 + result = [] + + # compute sel by hand for trlno in range(len(discrete.trials)): - thisTrial = discrete.trials[trlno][:, 0] - if isinstance(timeSel, list): + trlTime = discrete.time[trlno] + print('Test time', trlTime) + if timeSel is None or timeSel == "all": + idx = np.arange(trlTime.size).tolist() + else: if tselect == "toi": - trlRes = [] - for idx in smpIdx: - trlRes += list(np.where(thisTrial == idx + trlno * self.lenTrial)[0]) + idx = [] + for tp in timeSel: + idx.append(np.abs(trlTime - tp).argmin()) + # remove duplicates + arrayIdx = np.array(idx) + _, xdi = np.unique(arrayIdx.astype(np.intp), return_index=True) + arrayIdx = arrayIdx[np.sort(xdi)] + + # check for repeats + idx = [] + for closest in arrayIdx: + idx += np.where(trlTime == trlTime[closest])[0].tolist() else: - start = smpIdx[0] + trlno * self.lenTrial - stop = smpIdx[1] + trlno * self.lenTrial - candidates = np.intersect1d(thisTrial[thisTrial >= start], - thisTrial[thisTrial <= stop]) - trlRes = [] - for cand in candidates: - trlRes += list(np.where(thisTrial == cand)[0]) - else: - trlRes = slice(0, thisTrial.size, 1) + idx = np.intersect1d(np.where(trlTime >= timeSel[0])[0], + np.where(trlTime <= timeSel[1])[0]).tolist() + + # check that correct data was selected + print('Test idx', idx) + print('My idx', sel[trlno]) + assert np.array_equal(discrete.trials[trlno][idx, :], + discrete.trials[trlno][sel[trlno], :]) + if not isinstance(idx, slice) and len(idx) > 1: + timeSteps = np.diff(idx) + if timeSteps.min() == timeSteps.max() == 1: + idx = slice(idx[0], idx[-1] + 1, 1) + result.append(idx) + + # check correct format of selector (list -> slice etc.) + assert np.array_equal(result, sel) + + # perform actual data-selection and ensure identity of results + selected = selectdata(discrete, {tselect: timeSel}) + for trialno in range(len(discrete.trials)): + #discrIdx[timeIdx] = result[trialno] + assert np.array_equal(selected.trials[trialno], + discrete.trials[trialno][result[trialno],:])#[tuple(discrIdx)]) + + # for timeSel in selDict[tselect]: + # if isinstance(timeSel, list): + # smpIdx = [] + # for tp in timeSel: + # if np.isfinite(tp): + # smpIdx.append(np.abs(np.array(trlTime) - tp).argmin()) + # else: + # smpIdx.append(tp) + # result = [] + # sel = Selector(discrete, {tselect: timeSel}).time + # selected = selectdata(discrete, {tselect: timeSel}) + # tk = 0 + # for trlno in range(len(discrete.trials)): + # thisTrial = discrete.trials[trlno][:, 0] + # if isinstance(timeSel, list): + # if tselect == "toi": + # trlRes = [] + # for idx in smpIdx: + # trlRes += list(np.where(thisTrial == idx + trlno * self.lenTrial)[0]) + # else: + # start = smpIdx[0] + trlno * self.lenTrial + # stop = smpIdx[1] + trlno * self.lenTrial + # candidates = np.intersect1d(thisTrial[thisTrial >= start], + # thisTrial[thisTrial <= stop]) + # trlRes = [] + # for cand in candidates: + # trlRes += list(np.where(thisTrial == cand)[0]) + # else: + # trlRes = slice(0, thisTrial.size, 1) # ensure that actually selected data is correct - assert np.array_equal(discrete.trials[trlno][trlRes, :], - discrete.trials[trlno][sel[trlno], :]) - if sel[trlno]: - assert np.array_equal(selected.trials[tk], - discrete.trials[trlno][sel[trlno], :]) - tk += 1 - - if not isinstance(trlRes, slice) and len(trlRes) > 1: - sampSteps = np.diff(trlRes) - if sampSteps.min() == sampSteps.max() == 1: - trlRes = slice(trlRes[0], trlRes[-1] + 1, 1) - result.append(trlRes) + # assert np.array_equal(discrete.trials[trlno][trlRes, :], + # discrete.trials[trlno][sel[trlno], :]) + # if sel[trlno]: + # assert np.array_equal(selected.trials[tk], + # discrete.trials[trlno][sel[trlno], :]) + # tk += 1 + + # if not isinstance(trlRes, slice) and len(trlRes) > 1: + # sampSteps = np.diff(trlRes) + # if sampSteps.min() == sampSteps.max() == 1: + # trlRes = slice(trlRes[0], trlRes[-1] + 1, 1) + # result.append(trlRes) # check correct format of selector (list -> slice etc.) - assert result == sel + # assert result == sel def test_spectral_foifoilim(self): From c0281564a7a089aa28ea74af8e2f02385be3201b Mon Sep 17 00:00:00 2001 From: KatharineShapcott <65502584+KatharineShapcott@users.noreply.github.com> Date: Wed, 4 May 2022 08:55:48 +0200 Subject: [PATCH 19/38] FIX: removed unused code in test --- syncopy/tests/test_selectdata.py | 62 ++------------------------------ 1 file changed, 2 insertions(+), 60 deletions(-) diff --git a/syncopy/tests/test_selectdata.py b/syncopy/tests/test_selectdata.py index 4baabfc4e..d98251069 100644 --- a/syncopy/tests/test_selectdata.py +++ b/syncopy/tests/test_selectdata.py @@ -665,27 +665,19 @@ def test_discrete_toitoilim(self): [1.0, np.inf], # unbounded from above [-np.inf, 1.0])} # unbounded from below - # all trials have same time-scale for both `EventData` and `SpikeData`: take 1st one as reference - # trlTime = (np.arange(0, self.trl["SpikeData"][0, 1] - self.trl["SpikeData"][0, 0]) - # + self.trl["SpikeData"][0, 2]) / self.samplerate - # the below method of extracting spikes satisfying `toi`/`toilim` only works w/equidistant trials! for dclass in ["SpikeData", "EventData"]: discrete = getattr(spd, dclass)(data=self.data[dclass], trialdefinition=self.trl[dclass], samplerate=self.samplerate) - discrIdx = [slice(None)] * len(discrete.dimord) - #timeIdx = discrete.dimord.index("time") for tselect in ["toi", "toilim"]: for timeSel in selDict[tselect]: - print({tselect: timeSel}) sel = Selector(discrete, {tselect: timeSel}).time result = [] # compute sel by hand for trlno in range(len(discrete.trials)): trlTime = discrete.time[trlno] - print('Test time', trlTime) if timeSel is None or timeSel == "all": idx = np.arange(trlTime.size).tolist() else: @@ -707,8 +699,6 @@ def test_discrete_toitoilim(self): np.where(trlTime <= timeSel[1])[0]).tolist() # check that correct data was selected - print('Test idx', idx) - print('My idx', sel[trlno]) assert np.array_equal(discrete.trials[trlno][idx, :], discrete.trials[trlno][sel[trlno], :]) if not isinstance(idx, slice) and len(idx) > 1: @@ -723,57 +713,9 @@ def test_discrete_toitoilim(self): # perform actual data-selection and ensure identity of results selected = selectdata(discrete, {tselect: timeSel}) for trialno in range(len(discrete.trials)): - #discrIdx[timeIdx] = result[trialno] assert np.array_equal(selected.trials[trialno], - discrete.trials[trialno][result[trialno],:])#[tuple(discrIdx)]) - - # for timeSel in selDict[tselect]: - # if isinstance(timeSel, list): - # smpIdx = [] - # for tp in timeSel: - # if np.isfinite(tp): - # smpIdx.append(np.abs(np.array(trlTime) - tp).argmin()) - # else: - # smpIdx.append(tp) - # result = [] - # sel = Selector(discrete, {tselect: timeSel}).time - # selected = selectdata(discrete, {tselect: timeSel}) - # tk = 0 - # for trlno in range(len(discrete.trials)): - # thisTrial = discrete.trials[trlno][:, 0] - # if isinstance(timeSel, list): - # if tselect == "toi": - # trlRes = [] - # for idx in smpIdx: - # trlRes += list(np.where(thisTrial == idx + trlno * self.lenTrial)[0]) - # else: - # start = smpIdx[0] + trlno * self.lenTrial - # stop = smpIdx[1] + trlno * self.lenTrial - # candidates = np.intersect1d(thisTrial[thisTrial >= start], - # thisTrial[thisTrial <= stop]) - # trlRes = [] - # for cand in candidates: - # trlRes += list(np.where(thisTrial == cand)[0]) - # else: - # trlRes = slice(0, thisTrial.size, 1) - - # ensure that actually selected data is correct - # assert np.array_equal(discrete.trials[trlno][trlRes, :], - # discrete.trials[trlno][sel[trlno], :]) - # if sel[trlno]: - # assert np.array_equal(selected.trials[tk], - # discrete.trials[trlno][sel[trlno], :]) - # tk += 1 - - # if not isinstance(trlRes, slice) and len(trlRes) > 1: - # sampSteps = np.diff(trlRes) - # if sampSteps.min() == sampSteps.max() == 1: - # trlRes = slice(trlRes[0], trlRes[-1] + 1, 1) - # result.append(trlRes) - - # check correct format of selector (list -> slice etc.) - # assert result == sel - + discrete.trials[trialno][result[trialno],:]) + def test_spectral_foifoilim(self): # this selection only works w/the dummy frequency data constructed above!!! From cf18d5d5b7856392883ab992e4a93d93303a7499 Mon Sep 17 00:00:00 2001 From: Katharine Shapcott Date: Wed, 4 May 2022 10:24:38 +0200 Subject: [PATCH 20/38] FIX: added back in unused variable --- syncopy/tests/test_selectdata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/syncopy/tests/test_selectdata.py b/syncopy/tests/test_selectdata.py index d98251069..e02fc67b1 100644 --- a/syncopy/tests/test_selectdata.py +++ b/syncopy/tests/test_selectdata.py @@ -670,6 +670,7 @@ def test_discrete_toitoilim(self): discrete = getattr(spd, dclass)(data=self.data[dclass], trialdefinition=self.trl[dclass], samplerate=self.samplerate) + discrIdx = [slice(None)] * len(discrete.dimord) for tselect in ["toi", "toilim"]: for timeSel in selDict[tselect]: sel = Selector(discrete, {tselect: timeSel}).time From 319ff61684da74c57a7fd0dd2f1f7f52047ddc18 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 6 May 2022 11:17:56 +0200 Subject: [PATCH 21/38] FIX: Use updated version in setup - use hard-coded `releaseVersion` if setup.py is invoked on master branch instead of hoping for `scm_version` to work correctly - included `psutil` in dependency list On branch dev Changes to be committed: modified: setup.py modified: syncopy.yml --- setup.py | 21 +++++++++++---------- syncopy.yml | 1 + 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index f373aabaf..661872749 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ # Builtins import datetime from setuptools import setup +import subprocess # External packages import ruamel.yaml @@ -11,21 +12,21 @@ sys.path.insert(0, ".") from conda2pip import conda2pip -# Set release version by hand -releaseVersion = "0.21" +# Set release version by hand for master branch +releaseVersion = "0.22" # Get necessary and optional package dependencies required, dev = conda2pip(return_lists=True) -# Get package version for citationFile (for dev-builds this might differ from -# test-PyPI versions, which are ordered by recency) -version = get_version(root='.', relative_to=__file__, local_scheme="no-local-version") - -# Release versions (commits at tag) have suffix "dev0": use `releaseVersion` as -# fixed version. for TestPyPI uploads, keep the local `tag.devx` scheme -if version.split(".dev")[-1] == "0": - versionKws = {"use_scm_version" : False, "version" : releaseVersion} +# If code has not been obtained via `git` or we're inside the master branch, +# use the hard-coded `releaseVersion` as version. Otherwise keep the local `tag.devx` +# scheme for TestPyPI uploads +proc = subprocess.run("git branch --show-current", shell=True, capture_output=True, text=True) +if proc.returncode !=0 or proc.stdout.strip() == "master": + version = releaseVersion + versionKws = {"use_scm_version" : False, "version" : version} else: + version = get_version(root='.', relative_to=__file__, local_scheme="no-local-version") versionKws = {"use_scm_version" : {"local_scheme": "no-local-version"}} # Update citation file diff --git a/syncopy.yml b/syncopy.yml index 281ca59bb..0db9d776d 100644 --- a/syncopy.yml +++ b/syncopy.yml @@ -10,6 +10,7 @@ dependencies: - natsort - numpy >= 1.10, < 2.0 - pip + - psutil - python >= 3.8, < 3.9 - scipy >= 1.5 - tqdm >= 4.31 From 4b0debcffeb40a25ec0a7ae51f08975a3cb2e688 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 6 May 2022 13:57:56 +0200 Subject: [PATCH 22/38] NEW: Included custom-dimord EventData objects in selection tests - added `EventData` with custom dimord in `selectdata` tests On branch discrete-trialtime Changes to be committed: modified: syncopy/tests/test_selectdata.py --- syncopy/tests/test_selectdata.py | 39 ++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/syncopy/tests/test_selectdata.py b/syncopy/tests/test_selectdata.py index e02fc67b1..21035c9fc 100644 --- a/syncopy/tests/test_selectdata.py +++ b/syncopy/tests/test_selectdata.py @@ -330,6 +330,10 @@ class TestSelector(): data["EventData"][2::3, 1] = 2 trl["EventData"] = trl["AnalogData"] + # Append customized columns to EventData dataset + data["EventDataDimord"] = np.hstack([data["EventData"], data["EventData"]]) + trl["EventDataDimord"] = trl["AnalogData"] + # Define data classes to be used in tests below classes = ["AnalogData", "SpectralData", "SpikeData", "EventData"] @@ -337,11 +341,15 @@ class TestSelector(): def test_general(self): # construct expected results for `DiscreteData` objects defined above - mapDict = {"unit": "SpikeData", "eventid": "EventData"} - for prop, dclass in mapDict.items(): - discrete = getattr(spd, dclass)(data=self.data[dclass], + mapDict = {"SpikeData" : "unit", "EventData" : "eventid"} + for dset in ["SpikeData", "EventData", "EventDataDimord"]: + dclass = "".join(dset.partition("Data")[:2]) + prop = mapDict[dclass] + dimord = ["sample", "eventid", "custom1", "custom2"] if dset == "EventDataDimord" else None + discrete = getattr(spd, dclass)(data=self.data[dset], trialdefinition=self.trl[dclass], - samplerate=self.samplerate) + samplerate=self.samplerate, + dimord=dimord) propIdx = discrete.dimord.index(prop) # convert selection from `selectDict` to a usable integer-list @@ -424,12 +432,15 @@ def test_general(self): assert isinstance(ang.show(trials=0, toilim=[0, 1]), np.ndarray) assert isinstance(ang.show(trials=[0, 1], toi=[0, 1]), list) assert isinstance(ang.show(trials=[0, 1], toilim=[0, 1]), list) - + # go through all data-classes defined above - for dclass in self.classes: - dummy = getattr(spd, dclass)(data=self.data[dclass], + for dset in self.data.keys(): + dclass = "".join(dset.partition("Data")[:2]) + dimord = ["sample", "eventid", "custom1", "custom2"] if dset == "EventDataDimord" else None + dummy = getattr(spd, dclass)(data=self.data[dset], trialdefinition=self.trl[dclass], - samplerate=self.samplerate) + samplerate=self.samplerate, + dimord=dimord) # test trial selection selection = Selector(dummy, {"trials": [3, 1]}) @@ -657,7 +668,7 @@ def test_discrete_toitoilim(self): [2.0, 0.5, 2.5], # unsorted list [1.0, 0.5, 0.5, 1.5], # repetition [0.5, 0.5, 1.0, 1.5], # preserve repetition, don't convert to slice - [0.5, 1.0, 1.5]), # sorted list + [0.5, 1.0, 1.5]), # sorted list "toilim": (None, # trivial "selection" of entire contents "all", # trivial "selection" of entire contents [0.5, 1.5], # regular range @@ -666,10 +677,13 @@ def test_discrete_toitoilim(self): [-np.inf, 1.0])} # unbounded from below # the below method of extracting spikes satisfying `toi`/`toilim` only works w/equidistant trials! - for dclass in ["SpikeData", "EventData"]: + for dset in ["SpikeData", "EventData", "EventDataDimord"]: + dclass = "".join(dset.partition("Data")[:2]) + dimord = ["sample", "eventid", "custom1", "custom2"] if dset == "EventDataDimord" else None discrete = getattr(spd, dclass)(data=self.data[dclass], trialdefinition=self.trl[dclass], - samplerate=self.samplerate) + samplerate=self.samplerate, + dimord=dimord) discrIdx = [slice(None)] * len(discrete.dimord) for tselect in ["toi", "toilim"]: for timeSel in selDict[tselect]: @@ -713,10 +727,11 @@ def test_discrete_toitoilim(self): # perform actual data-selection and ensure identity of results selected = selectdata(discrete, {tselect: timeSel}) + assert selected.dimord == discrete.dimord for trialno in range(len(discrete.trials)): assert np.array_equal(selected.trials[trialno], discrete.trials[trialno][result[trialno],:]) - + def test_spectral_foifoilim(self): # this selection only works w/the dummy frequency data constructed above!!! From d1e398062732055db7cf74720fed93d8b8d57744 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Mon, 9 May 2022 14:55:11 +0200 Subject: [PATCH 23/38] NEW: Included tests for EventData w/non-standard columns - amended tests with `EventData` objects that use custom column names On branch discrete-trialtime Changes to be committed: modified: syncopy/tests/test_discretedata.py modified: syncopy/tests/test_selectdata.py --- syncopy/tests/test_discretedata.py | 47 +++++++++++++++++++++++------- syncopy/tests/test_selectdata.py | 10 +++---- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/syncopy/tests/test_discretedata.py b/syncopy/tests/test_discretedata.py index 1cadec4c3..827e6a018 100644 --- a/syncopy/tests/test_discretedata.py +++ b/syncopy/tests/test_discretedata.py @@ -256,12 +256,14 @@ class TestEventData(): data2 = data.copy() data2[:, -1] = data[:, 0] data2[:, 0] = data[:, -1] + data3 = np.hstack([data2, data2]) trl = np.vstack([np.arange(0, ns, 5), np.arange(5, ns + 5, 5), np.ones((int(ns / 5), )), np.ones((int(ns / 5), )) * np.pi]).T num_smp = np.unique(data[:, 0]).size num_evt = np.unique(data[:, 1]).size + customDimord = ["sample", "eventid", "custom1", "custom2"] adata = np.arange(1, nc * ns + 1).reshape(ns, nc) @@ -298,7 +300,7 @@ def test_ed_trialretrieval(self): trl_ref = self.data[idx, ...] assert np.array_equal(dummy._get_trial(trlno), trl_ref) - # test ``_get_trial`` with NumPy array: swapped dimensions + # test `_get_trial` with NumPy array: swapped dimensions dummy = EventData(self.data2, trialdefinition=self.trl, dimord=["eventid", "sample"]) smp = self.data2[:, -1] @@ -308,6 +310,18 @@ def test_ed_trialretrieval(self): trl_ref = self.data2[idx, ...] assert np.array_equal(dummy._get_trial(trlno), trl_ref) + # test `_get_trial` with NumPy array: customized columns names + nuDimord = ["eventid", "sample", "custom1", "custom2"] + dummy = EventData(self.data3, trialdefinition=self.trl, + dimord=nuDimord) + assert dummy.dimord == nuDimord + smp = self.data3[:, -1] + for trlno, start in enumerate(range(0, self.ns, 5)): + idx = np.intersect1d(np.where(smp >= start)[0], + np.where(smp < start + 5)[0]) + trl_ref = self.data3[idx, ...] + assert np.array_equal(dummy._get_trial(trlno), trl_ref) + def test_ed_saveload(self): with tempfile.TemporaryDirectory() as tdir: fname = os.path.join(tdir, "dummy") @@ -354,6 +368,16 @@ def test_ed_saveload(self): assert dummy2.dimord == dummy.dimord assert dummy2.eventid.size == self.num_smp # swapped assert dummy2.data.shape == dummy.data.shape + del dummy, dummy2 + + # save dataset w/custom column names and ensure `dimord` is preserved + dummy = EventData(np.hstack([self.data, self.data]), dimord=self.customDimord, samplerate=10) + dummy.save(fname + "_customDimord") + filename = construct_spy_filename(fname + "_customDimord", dummy) + dummy2 = load(filename) + assert dummy2.dimord == dummy.dimord + assert dummy2.eventid.size == self.num_evt + assert dummy2.data.shape == dummy.data.shape # Delete all open references to file objects b4 closing tmp dir del dummy, dummy2 @@ -388,9 +412,10 @@ def test_ed_trialsetting(self): samples = np.arange(0, int(self.ns / 3), 3)[1:] dappend = np.vstack([samples, np.full(samples.shape, 2)]).T data3 = np.vstack([self.data, dappend]) + data3 = np.hstack([data3, data3]) idx = np.argsort(data3[:, 0]) data3 = data3[idx, :] - evt_dummy = EventData(data3, samplerate=sr_e) + evt_dummy = EventData(data3, dimord=self.customDimord, samplerate=sr_e) evt_dummy.definetrial(start=0, stop=1) assert np.array_equal(sinfo2, evt_dummy.sampleinfo) @@ -410,7 +435,7 @@ def test_ed_trialsetting(self): dcodes = dcodes[idx + 1:] dsamps = dsamps[idx + 1:] sinfo3[sk, :] = [start, stop] - evt_dummy = EventData(data3, samplerate=sr_e) + evt_dummy = EventData(data3, dimord=self.customDimord, samplerate=sr_e) evt_dummy.definetrial(start=[2, 2, 1], stop=[1, 2, 0]) assert np.array_equal(evt_dummy.sampleinfo, sinfo3) @@ -420,7 +445,7 @@ def test_ed_trialsetting(self): ang_dummy = AnalogData(self.adata, samplerate=sr_a) ang_dummy.definetrial(evt_dummy) assert np.array_equal(ang_dummy.sampleinfo, sinfo_a) - evt_dummy = EventData(data=data3, samplerate=sr_e) + evt_dummy = EventData(data=data3, dimord=self.customDimord, samplerate=sr_e) evt_dummy.definetrial(pre=pre, post=post, trigger=1) ang_dummy.definetrial(evt_dummy) assert np.array_equal(ang_dummy.sampleinfo, sinfo_a) @@ -430,17 +455,18 @@ def test_ed_trialsetting(self): ang_dummy = AnalogData(self.adata, samplerate=sr_a) ang_dummy.definetrial(evt_dummy, pre=pre, post=post, trigger=1) assert np.array_equal(ang_dummy.sampleinfo, sinfo_a) - evt_dummy = EventData(data=data3, samplerate=sr_e) + evt_dummy = EventData(data=data3, dimord=self.customDimord, samplerate=sr_e) ang_dummy = AnalogData(self.adata, samplerate=sr_a) ang_dummy.definetrial(evt_dummy, pre=pre, post=post, trigger=1) assert np.array_equal(ang_dummy.sampleinfo, sinfo_a) - # Extend data and provoke an exception due to out of bounds erro + # Extend data and provoke an exception due to out of bounds error smp = np.vstack([np.arange(self.ns, int(2.5 * self.ns), 5), np.zeros((int((1.5 * self.ns) / 5),))]).T smp[1::2, 1] = 1 + smp = np.hstack([smp, smp]) data4 = np.vstack([data3, smp]) - evt_dummy = EventData(data=data4, samplerate=sr_e) + evt_dummy = EventData(data=data4, dimord=self.customDimord, samplerate=sr_e) evt_dummy.definetrial(pre=pre, post=post, trigger=1) # with pytest.raises(SPYValueError): # ang_dummy.definetrial(evt_dummy) @@ -452,7 +478,7 @@ def test_ed_trialsetting(self): # We need `clip_edges` to make trial-definition work data4 = data4[:-2, :] data4[-2, 0] = data4[-1, 0] - evt_dummy = EventData(data=data4, samplerate=sr_e) + evt_dummy = EventData(data=data4, dimord=self.customDimord, samplerate=sr_e) evt_dummy.definetrial(pre=pre, post=post, trigger=1) # with pytest.raises(SPYValueError): # ang_dummy.definetrial(evt_dummy) @@ -495,10 +521,11 @@ def test_ed_trialsetting(self): def test_ed_dataselection(self, fulltests): # Create testing objects (regular and swapped dimords) - dummy = EventData(data=self.data, + dummy = EventData(data=np.hstack([self.data, self.data]), + dimord=self.customDimord, trialdefinition=self.trl, samplerate=2.0) - ymmud = EventData(data=self.data[:, ::-1], + ymmud = EventData(data=np.hstack([self.data[:, ::-1], self.data[:, ::-1]]), trialdefinition=self.trl, samplerate=2.0, dimord=dummy.dimord[::-1]) diff --git a/syncopy/tests/test_selectdata.py b/syncopy/tests/test_selectdata.py index 21035c9fc..ea7cc7416 100644 --- a/syncopy/tests/test_selectdata.py +++ b/syncopy/tests/test_selectdata.py @@ -333,6 +333,7 @@ class TestSelector(): # Append customized columns to EventData dataset data["EventDataDimord"] = np.hstack([data["EventData"], data["EventData"]]) trl["EventDataDimord"] = trl["AnalogData"] + customEvtDimord = ["sample", "eventid", "custom1", "custom2"] # Define data classes to be used in tests below classes = ["AnalogData", "SpectralData", "SpikeData", "EventData"] @@ -345,7 +346,7 @@ def test_general(self): for dset in ["SpikeData", "EventData", "EventDataDimord"]: dclass = "".join(dset.partition("Data")[:2]) prop = mapDict[dclass] - dimord = ["sample", "eventid", "custom1", "custom2"] if dset == "EventDataDimord" else None + dimord = self.customEvtDimord if dset == "EventDataDimord" else None discrete = getattr(spd, dclass)(data=self.data[dset], trialdefinition=self.trl[dclass], samplerate=self.samplerate, @@ -436,7 +437,7 @@ def test_general(self): # go through all data-classes defined above for dset in self.data.keys(): dclass = "".join(dset.partition("Data")[:2]) - dimord = ["sample", "eventid", "custom1", "custom2"] if dset == "EventDataDimord" else None + dimord = self.customEvtDimord if dset == "EventDataDimord" else None dummy = getattr(spd, dclass)(data=self.data[dset], trialdefinition=self.trl[dclass], samplerate=self.samplerate, @@ -679,12 +680,11 @@ def test_discrete_toitoilim(self): # the below method of extracting spikes satisfying `toi`/`toilim` only works w/equidistant trials! for dset in ["SpikeData", "EventData", "EventDataDimord"]: dclass = "".join(dset.partition("Data")[:2]) - dimord = ["sample", "eventid", "custom1", "custom2"] if dset == "EventDataDimord" else None - discrete = getattr(spd, dclass)(data=self.data[dclass], + dimord = self.customEvtDimord if dset == "EventDataDimord" else None + discrete = getattr(spd, dclass)(data=self.data[dset], trialdefinition=self.trl[dclass], samplerate=self.samplerate, dimord=dimord) - discrIdx = [slice(None)] * len(discrete.dimord) for tselect in ["toi", "toilim"]: for timeSel in selDict[tselect]: sel = Selector(discrete, {tselect: timeSel}).time From b2f8fcf24927646c2bdf2066115e5dd30505e832 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Mon, 9 May 2022 14:55:52 +0200 Subject: [PATCH 24/38] CHG: Check consistency of custom EventData dimords - ensure that provided custom `dimord` of an `EventData` object matches up with actual data attached (e.g., a six-element dimord requires an `(N, 6)` array) On branch discrete-trialtime Changes to be committed: modified: syncopy/datatype/base_data.py --- syncopy/datatype/base_data.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/syncopy/datatype/base_data.py b/syncopy/datatype/base_data.py index 4b7c83c39..89f016778 100644 --- a/syncopy/datatype/base_data.py +++ b/syncopy/datatype/base_data.py @@ -250,11 +250,15 @@ def _set_dataset_property_with_ndarray(self, inData, propertyName, ndim): Number of expected array dimensions. """ + # Ensure array has right no. of dimensions try: array_parser(inData, varname="data", dims=ndim) except Exception as exc: raise exc + # Gymnastics for `DiscreteData` objects w/non-standard `dimord`s + self._check_dataset_property_discretedata(inData) + # If there is existing data, replace values if shape and type match if isinstance(getattr(self, "_" + propertyName), (np.memmap, h5py.Dataset)): prop = getattr(self, "_" + propertyName) @@ -303,6 +307,7 @@ def _set_dataset_property_with_memmap(self, inData, propertyName, ndim): lgl = "{}-dimensional data".format(ndim) act = "{}-dimensional memmap".format(inData.ndim) raise SPYValueError(legal=lgl, varname=propertyName, actual=act) + self._check_dataset_property_discretedata(inData) self.mode = inData.mode self.filename = inData.filename @@ -326,14 +331,18 @@ def _set_dataset_property_with_dataset(self, inData, propertyName, ndim): act = "backing HDF5 file is closed" raise SPYValueError(legal=lgl, actual=act, varname="data") - self._mode = inData.file.mode - self.filename = inData.file.filename - + # Ensure dataset has right no. of dimensions if inData.ndim != ndim: lgl = "{}-dimensional data".format(ndim) act = "{}-dimensional HDF5 dataset or memmap".format(inData.ndim) raise SPYValueError(legal=lgl, varname="data", actual=act) + # Gymnastics for `DiscreteData` objects w/non-standard `dimord`s + self._check_dataset_property_discretedata(inData) + + self._mode = inData.file.mode + self.filename = inData.file.filename + setattr(self, "_" + propertyName, inData) def _set_dataset_property_with_list(self, inData, propertyName, ndim): @@ -381,7 +390,7 @@ def _set_dataset_property_with_list(self, inData, propertyName, ndim): else: # EventData nCol = inData[0].shape[1] if any(val.shape[1] != nCol for val in inData): - lgl = "NumPy 2d-arrays with 3 columns" + lgl = "NumPy 2d-arrays with {} columns".format(nCol) act = "NumPy arrays of different shape" raise SPYValueError(legal=lgl, varname="data", actual=act) trialLens = [np.nanmax(val[:, self.dimord.index("sample")]) for val in inData] @@ -415,6 +424,24 @@ def _set_dataset_property_with_list(self, inData, propertyName, ndim): self._set_dataset_property_with_ndarray(data, propertyName, ndim) self.trialdefinition = trialdefinition + def _check_dataset_property_discretedata(self, inData): + """Check `DiscreteData` input data for shape consistency + + Parameters + ---------- + inData : array/memmap/h5py.Dataset + array-like to be stored as a `DiscreteData` data source + """ + + # Special case `DiscreteData`: `dimord` encodes no. of expected cols/rows; + # ensure this is consistent w/`inData`! + if any(["DiscreteData" in str(base) for base in self.__class__.__mro__]): + if len(self._defaultDimord) not in inData.shape: + lgl = "array with {} columns corresponding to dimord {}" + lgl = lgl.format(len(self._defaultDimord), self._defaultDimord) + act = "array with shape {}".format(str(inData.shape)) + raise SPYValueError(legal=lgl, varname="data", actual=act) + def _is_empty(self): return all([getattr(self, attr) is None for attr in self._hdfFileDatasetProperties]) From 1ac80a3272836b27a4edb95289f90fb92e5e7416 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Mon, 9 May 2022 14:56:26 +0200 Subject: [PATCH 25/38] FIX: Changed dimord setup when loading spy containers - do not assign `dimord` to the `out` object: instead pass it to the class constructor (if `new_out` is `True`). Only set dimord for provided `out` datasets. On branch discrete-trialtime Changes to be committed: modified: syncopy/io/load_spy_container.py --- syncopy/io/load_spy_container.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/syncopy/io/load_spy_container.py b/syncopy/io/load_spy_container.py index a14ae5226..4bb021d62 100644 --- a/syncopy/io/load_spy_container.py +++ b/syncopy/io/load_spy_container.py @@ -276,20 +276,18 @@ def _load(filename, checksum, mode, out): actual=hsh_msg.format(hsh=hsh)) # Parsing is done, create new or check provided object + dimord = jsonDict.pop("dimord") if out is not None: try: data_parser(out, varname="out", writable=True, dataclass=jsonDict["dataclass"]) except Exception as exc: raise exc new_out = False + out.dimord = dimord else: - out = dataclass() + out = dataclass(dimord=dimord) new_out = True - # First and foremost, assign dimensional information - dimord = jsonDict.pop("dimord") - out.dimord = dimord - # Access data on disk (error checking is done by setters) out.mode = mode for datasetProperty in out._hdfFileDatasetProperties: From 4c72995961aa48faf372e43ba783a9fc18eb8c10 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Mon, 9 May 2022 16:20:25 +0200 Subject: [PATCH 26/38] CHG: Updated changelog - included most recent feature additions in CHANGELOG.md On branch dev Changes to be committed: modified: CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a235475b3..ad0c27042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ## [Unreleased] ### NEW +### NEW +- Added support for flexible columns in `EventData` (thanks to @KatharineShapcott) ### CHANGED @@ -19,6 +21,7 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ### FIXED - Improved legibility of `spy.__version__` for non-release installations - Correctly process equidistant `toi` arrays with large spacing in `freqanalysis` +- Corrected `trialtime` for `DiscreteData` objects (thanks to @KatharineShapcott) ## [v0.21] - 2022-04-13 Feature update and bugfixes. From a807d1e95d7102b4f510b07691b08646c14b11b7 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 10 May 2022 14:30:03 +0200 Subject: [PATCH 27/38] FIX: Houskeeping - removed commented-out obsolete tests On branch dev Changes to be committed: modified: syncopy/tests/test_specest.py --- syncopy/tests/test_specest.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/syncopy/tests/test_specest.py b/syncopy/tests/test_specest.py index c50cdc5b6..ecf2ce1d4 100644 --- a/syncopy/tests/test_specest.py +++ b/syncopy/tests/test_specest.py @@ -676,9 +676,6 @@ def test_tf_toi(self): toiArrs = [np.arange(-2,7), np.arange(-1, 6, 1/self.tfData.samplerate), np.arange(1, 6, 2)] - # toiArrs = [np.arange(-10, 15.1), - # np.arange(-15, -10, 1/self.tfData.samplerate), - # np.arange(1, 20, 2)] winSizes = [0.5, 1.0] # Combine `toi`-testing w/in-place data-pre-selection @@ -1360,9 +1357,6 @@ def test_slet_toi(self, fulltests): toiArrs = [np.arange(-2,7), np.arange(-1, 6, 1/self.tfData.samplerate), np.arange(1, 6, 2)] - # toiArrs = [np.arange(-10, 15.1), - # np.arange(-15, -10, 1/self.tfData.samplerate), - # np.arange(1, 20, 2)] # Just pick one `toi` at random for quickly running tests if not fulltests: From 464374cf1f09f9fca232ddf22ecae4720fa675e3 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Tue, 10 May 2022 14:30:41 +0200 Subject: [PATCH 28/38] FIX: Bugfix in best_match - account for the possibility that `source` might be a singleton array On branch dev Changes to be committed: modified: syncopy/shared/tools.py --- syncopy/shared/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncopy/shared/tools.py b/syncopy/shared/tools.py index a246f1070..dcbdb05cf 100644 --- a/syncopy/shared/tools.py +++ b/syncopy/shared/tools.py @@ -147,7 +147,7 @@ def best_match(source, selection, span=False, tol=None, squash_duplicates=False) np.where(source <= selection[1])[0]) else: issorted = True - if np.diff(source).min() < 0: + if source.size > 1 and np.diff(source).min() < 0: issorted = False orig = np.array(source, copy=True) idx_orig = np.argsort(orig) From 71390f24596026ada4d159ace78de7759e497b5b Mon Sep 17 00:00:00 2001 From: tensionhead Date: Tue, 10 May 2022 14:38:18 +0200 Subject: [PATCH 29/38] CHG: Adapt to current FT spectral normalization - verified that for `hann`, multi-tapering and no tapering the fft (mtmfft and mtmconvol) results adhere to FTs current normalization - it is A**2 / 2 for a pure untapered harmonic, tapering distributes the power evenly such that the sum of the power recovers A**2 / 2 - padding attenuates the original power, as the total power gets normalized by the length after padding. This is NOT intended and FT will/might change that in the future! On branch FT-normalization Changes to be committed: modified: syncopy/specest/_norm_spec.py modified: syncopy/specest/mtmfft.py modified: syncopy/specest/stft.py --- syncopy/specest/_norm_spec.py | 11 ++++++----- syncopy/specest/mtmfft.py | 13 +++++++------ syncopy/specest/stft.py | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/syncopy/specest/_norm_spec.py b/syncopy/specest/_norm_spec.py index ea46173f7..2509bcd64 100644 --- a/syncopy/specest/_norm_spec.py +++ b/syncopy/specest/_norm_spec.py @@ -10,7 +10,7 @@ def _norm_spec(ftr, nSamples, fs, mode='bins'): """ Normalizes the complex Fourier transform to - power spectral density or dimensionless bin units. + power spectral density or 1Hz-bin units. """ # frequency bins @@ -29,16 +29,17 @@ def _norm_taper(taper, windows, nSamples): """ Helper function to normalize tapers such that the resulting spectra are normalized - to power density units. + with respect to the sum of the window. Meaning + that the total original (untapered) power gets + distributed over the spectral window response. """ if taper == 'dpss': windows *= np.sqrt(nSamples) - # only for padding - if taper == 'boxcar': + elif taper == 'boxcar': windows *= np.sqrt(nSamples / windows.sum()) # weird 3 point normalization, - # checks out (almost) exactly for 'hann' though + # checks out exactly for 'hann' though else: windows *= np.sqrt(4 / 3) * np.sqrt(nSamples / windows.sum()) diff --git a/syncopy/specest/mtmfft.py b/syncopy/specest/mtmfft.py index 11b6d5914..2a9d2af0c 100644 --- a/syncopy/specest/mtmfft.py +++ b/syncopy/specest/mtmfft.py @@ -31,7 +31,7 @@ def mtmfft(data_arr, Samplerate in Hz nSamples : int or None Absolute length of the (potentially to be padded) signals - or `None` for no padding (`N` is the number of samples) + or `None` for no padding. taper : str or None Taper function to use, one of `scipy.signal.windows` Set to `None` for no tapering. @@ -58,10 +58,9 @@ def mtmfft(data_arr, ``Sxx = np.real(ftr * ftr.conj()).mean(axis=0)`` - The FFT result is normalized such that this yields the power - spectral density. For a clean harmonic and a Fourier frequency bin - width of `dF` this will give a peak power of `A**2 / 2 * dF`, - with `A` as harmonic ampltiude. + The FFT result is normalized such that this yields the + spectral power. For a clean harmonic this will give a + peak power of `A**2 / 2`, with `A` as harmonic ampltiude. """ # attach dummy channel axis in case only a @@ -69,6 +68,7 @@ def mtmfft(data_arr, if data_arr.ndim < 2: data_arr = data_arr[:, np.newaxis] + # raw length without padding signal_length = data_arr.shape[0] if nSamples is None: nSamples = signal_length @@ -89,7 +89,7 @@ def mtmfft(data_arr, # only really 2d if taper='dpss' with Kmax > 1 # here we take the actual signal lengths! windows = np.atleast_2d(taper_func(signal_length, **taper_opt)) - # normalize window with length after padding + # normalize window with total (after padding) length windows = _norm_taper(taper, windows, nSamples) # Fourier transforms (nTapers x nFreq x nChannels) @@ -102,6 +102,7 @@ def mtmfft(data_arr, if demean_taper: win -= win.mean(axis=0) ftr[taperIdx] = np.fft.rfft(win, n=nSamples, axis=0) + # FT uses potentially padded length `nSamples` ftr[taperIdx] = _norm_spec(ftr[taperIdx], nSamples, samplerate) return ftr, freqs diff --git a/syncopy/specest/stft.py b/syncopy/specest/stft.py index 7730ee7b1..93eb00ce4 100644 --- a/syncopy/specest/stft.py +++ b/syncopy/specest/stft.py @@ -142,7 +142,7 @@ def stft(dat, # the complex transforms ftr = np.fft.rfft(dat, axis=-1) - # normalization to squared amplitude density + # normalization to power -> squared amplitude / 2 ftr = _norm_spec(ftr, nperseg, fs) # Roll frequency axis back to axis where the data came from From e91a36635cff6751c37a007999e0b45a3e82dfea Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 11 May 2022 14:13:32 +0200 Subject: [PATCH 30/38] FIX: MTMconvol tests - multi-tapering was done erroneously with half-sided tapsmofrq, now the perfect (rectangular) spectral smoothing in Hz gives the number of bins the original power spreads out On branch FT-normalization Your branch is up to date with 'origin/FT-normalization'. Changes to be committed: modified: syncopy/tests/backend/test_timefreq.py --- syncopy/tests/backend/test_timefreq.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/syncopy/tests/backend/test_timefreq.py b/syncopy/tests/backend/test_timefreq.py index 8c26b5722..29ff6a760 100644 --- a/syncopy/tests/backend/test_timefreq.py +++ b/syncopy/tests/backend/test_timefreq.py @@ -83,7 +83,7 @@ def test_mtmconvol(): # the transforms have shape (nTime, nTaper, nFreq, nChannel) ftr, freqs = mtmconvol.mtmconvol(signal, - samplerate=fs, taper='cosine', + samplerate=fs, taper='hann', nperseg=window_size, noverlap=window_size - 1) @@ -113,7 +113,7 @@ def test_mtmconvol(): origin='lower', extent=extent, vmin=0, - vmax=.5 * A**2 / df) + vmax=.5 * A**2) # zoom into foi region ax2.set_ylim((foi[0], foi[-1])) @@ -137,7 +137,7 @@ def test_mtmconvol(): c='0.5') # integrated power at the respective frquency - cycle_num = (spec[:, idx] * df > .5 * A**2 / np.e**2).sum() / fs * frequency + cycle_num = (spec[:, idx] > .5 * A**2 / np.e**2).sum() / fs * frequency print(f'{cycle_num} cycles for the {frequency} Hz band') # we have 2 times the cycles for each frequency (temporal neighbor) assert cycle_num > 2 * cycles @@ -151,13 +151,14 @@ def test_mtmconvol(): # ------------------------- taper = 'dpss' - tapsmofrq = 10 # Hz + tapsmofrq = 5 # two-sided in Hz # set parameters for scipy.signal.windows.dpss - NW = tapsmofrq * window_size / (2 * fs) + NW = tapsmofrq * window_size / fs # from the minBw setting NW always is at least 1 Kmax = int(2 * NW - 1) # optimal number of tapers taper_opt = {'Kmax': Kmax, 'NW': NW} + print(taper_opt) # the transforms have shape (nTime, nTaper, nFreq, nChannel) ftr2, freqs2 = mtmconvol.mtmconvol(signal, samplerate=fs, taper=taper, taper_opt=taper_opt, @@ -186,7 +187,7 @@ def test_mtmconvol(): origin='lower', extent=extent, vmin=0, - vmax=.5 * A**2 / df) + vmax=.5 * A**2) # zoom into foi region ax2.set_ylim((foi[0], foi[-1])) @@ -210,9 +211,8 @@ def test_mtmconvol(): # due to too much spectral broadening/smearing # so we just check that the maximum estimated # power within one bin is within 15% bounds of the real power - nBins = tapsmofrq / df - assert 0.4 * A**2 / df < spec2.max() * nBins < .65 * A**2 / df - + nBins = tapsmofrq + assert 0.4 * A**2 < spec2.max() * nBins < .65 * A**2 def test_superlet(): @@ -387,7 +387,7 @@ def test_mtmfft(): dpss_powers = dpss_spec[:, 0] # only 1 channel # check for integrated power (and taper normalisation) # summing up all dpss powers should give total power of the - # test signal which is A1**2 + A2**2 + # test signal which is (A1**2 + A2**2) / 2 assert np.allclose(np.sum(dpss_powers) * 2, A1**2 + A2**2, atol=1e-2) ax.plot(freqs[:150], dpss_powers[:150], label="Slepian", lw=2) From 89b2755e01159e47692bb04ad50daece61346f07 Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 11 May 2022 15:02:56 +0200 Subject: [PATCH 31/38] CHG: Padding interface now FT compatible - in FT the keyword is simply `pad`, and three modes are supported: 'maxperlen' (default, do nothing or pad to longest trial.. we had `None` before) float - padding length in seconds (we had in samples before) 'nextpow2' - nothing changed On branch FT-padding Changes to be committed: modified: syncopy/nwanalysis/connectivity_analysis.py modified: syncopy/shared/const_def.py modified: syncopy/shared/input_processors.py modified: syncopy/specest/freqanalysis.py modified: syncopy/tests/helpers.py modified: syncopy/tests/test_connectivity.py modified: syncopy/tests/test_specest.py --- syncopy/nwanalysis/connectivity_analysis.py | 38 +++++++-------- syncopy/shared/const_def.py | 2 +- syncopy/shared/input_processors.py | 52 ++++++++++++--------- syncopy/specest/freqanalysis.py | 28 +++++------ syncopy/tests/helpers.py | 18 +++---- syncopy/tests/test_connectivity.py | 22 ++++----- syncopy/tests/test_specest.py | 6 +-- 7 files changed, 85 insertions(+), 81 deletions(-) diff --git a/syncopy/nwanalysis/connectivity_analysis.py b/syncopy/nwanalysis/connectivity_analysis.py index 808536dfe..fe7df9abb 100644 --- a/syncopy/nwanalysis/connectivity_analysis.py +++ b/syncopy/nwanalysis/connectivity_analysis.py @@ -35,7 +35,7 @@ @unwrap_select @detect_parallel_client def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", - foi=None, foilim=None, pad_to_length=None, + foi=None, foilim=None, pad='maxperlen', polyremoval=None, tapsmofrq=None, nTaper=None, taper="hann", taper_opt=None, out=None, **kwargs): @@ -60,7 +60,7 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", * **taper** : one of :data:`~syncopy.shared.const_def.availableTapers` * **tapsmofrq** : spectral smoothing box for slepian tapers (in Hz) * **nTaper** : (optional) number of orthogonal tapers for slepian tapers - * **pad_to_length**: either pad to an absolute length or set to `'nextpow2'` + * **pad**: either pad to an absolute length in seconds or set to `'nextpow2'` "corr" : Cross-correlations Computes the one sided (positive lags) cross-correlations @@ -77,7 +77,7 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", * **taper** : one of :data:`~syncopy.shared.const_def.availableTapers` * **tapsmofrq** : spectral smoothing box for slepian tapers (in Hz) * **nTaper** : (optional, not recommended) number of slepian tapers - * **pad_to_length**: either pad to an absolute length or set to `'nextpow2'` + * **pad**: either pad to an absolute length in seconds or set to `'nextpow2'` Parameters ---------- @@ -101,17 +101,15 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", Frequency-window ``[fmin, fmax]`` (in Hz) of interest. The `foi` array will be constructed in 1Hz steps from `fmin` to `fmax` (inclusive). - pad_to_length : int, None or 'nextpow2' - Padding of the (tapered) signal, if set to a number pads all trials - to this absolute length. E.g. `pad_to_length=2000` pads all - trials to 2000 samples, if and only if the longest trial is - at maximum 2000 samples. - - Alternatively if all trials have the same initial lengths - setting `pad_to_length='nextpow2'` pads all trials to - the next power of two. - If `None` and trials have unequal lengths all trials are padded to match - the longest trial. + pad : 'maxperlen', float or 'nextpow2' + Padding of the input data, if set to a number pads all trials + to this absolute length in seconds. For instance ``pad=2`` pads all + trials to an absolute length of 2000 samples, if and only if the longest + trial contains at maximum 2000 samples and the samplerate is 1kHz. + Alternatively `pad='nextpow2'` pads all trials to + the next power of two of the longest trial. + For the default `'maxperlen'` and trials have unequal lengths all + trials are padded to match the longest trial. tapsmofrq : float or None Only valid if `method` is `'coh'` or `'granger'`. Enables multi-tapering and sets the amount of spectral @@ -177,13 +175,13 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", # --- Padding --- - if method == "corr" and pad_to_length: - lgl = "`None`, no padding needed/allowed for cross-correlations" - actual = f"{pad_to_length}" - raise SPYValueError(legal=lgl, varname="pad_to_length", actual=actual) + if method == "corr" and pad != 'maxperlen': + lgl = "'maxperlen', no padding needed/allowed for cross-correlations" + actual = f"{pad}" + raise SPYValueError(legal=lgl, varname="pad", actual=actual) # the actual number of samples in case of later padding - nSamples = process_padding(pad_to_length, lenTrials) + nSamples = process_padding(pad, lenTrials, data.samplerate) # --- Basic foi sanitization --- @@ -199,7 +197,7 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", "output": output, "keeptrials": keeptrials, "polyremoval": polyremoval, - "pad_to_length": pad_to_length} + "pad": pad} # --- Setting up specific Methods --- if method == 'granger': diff --git a/syncopy/shared/const_def.py b/syncopy/shared/const_def.py index 9fe92a8f0..c16862891 100644 --- a/syncopy/shared/const_def.py +++ b/syncopy/shared/const_def.py @@ -26,7 +26,7 @@ all_windows.remove("dpss") # activated via `tapsmofrq` availableTapers = all_windows -availablePaddingOpt = [None, 'nextpow2'] +availablePaddingOpt = ['maxperlen', 'nextpow2'] #: general, method agnostic, parameters for our CRs generalParameters = ("method", "output", "keeptrials", "samplerate", diff --git a/syncopy/shared/input_processors.py b/syncopy/shared/input_processors.py index d66889f08..8b92fb7e0 100644 --- a/syncopy/shared/input_processors.py +++ b/syncopy/shared/input_processors.py @@ -19,61 +19,67 @@ from syncopy.datatype.methods.padding import _nextpow2 -def process_padding(pad_to_length, lenTrials): +def process_padding(pad, lenTrials, samplerate): """ Simplified padding interface, for all taper based methods - padding has to be done **before** tapering! + padding has to be done **after** tapering! + + This function returns a number indicating the total + length in #samples of all trials after padding. Parameters ---------- - pad_to_length : int, None or 'nextpow2' - Either an integer indicating the absolute length of - the trials after padding or `'nextpow2'` to pad all trials - to the nearest power of two. If `None`, no padding is to - be performed + pad : 'maxperlen', float or 'nextpow2' + For the frontend default `maxperlen`, no padding is to + be performed in case of equal length trials but unequal lengths + trials get padded to the max. trial length. + A float indicates the absolute length of + all trials after padding in seconds. `'nextpow2'` pads all trials + to the nearest power of two. lenTrials : sequence of int_like Sequence holding all individual trial lengths + samplerate : float + The sampling rate in Hz Returns ------- abs_pad : int - Absolute length of all trials after padding + Absolute length of all trials after padding in #samples """ # supported padding options not_valid = False - if not isinstance(pad_to_length, (numbers.Number, str, type(None))): + if not isinstance(pad, (numbers.Number, str)): not_valid = True - elif isinstance(pad_to_length, str) and pad_to_length not in availablePaddingOpt: + elif isinstance(pad, str) and pad not in availablePaddingOpt: not_valid = True # bool is an int subclass, have to check for it separately... - if isinstance(pad_to_length, bool): + if isinstance(pad, bool): not_valid = True if not_valid: - lgl = "`None`, 'nextpow2' or an integer like number" - actual = f"{pad_to_length}" - raise SPYValueError(legal=lgl, varname="pad_to_length", actual=actual) + lgl = "'maxperlen', 'nextpow2' or a float number" + actual = f"{pad}" + raise SPYValueError(legal=lgl, varname="pad", actual=actual) # zero padding of ALL trials the same way - if isinstance(pad_to_length, numbers.Number): + if isinstance(pad, numbers.Number): - scalar_parser(pad_to_length, - varname='pad_to_length', - ntype='int_like', - lims=[lenTrials.max(), np.inf]) - abs_pad = pad_to_length + scalar_parser(pad, + varname='pad', + lims=[lenTrials.max() / samplerate, np.inf]) + abs_pad = int(pad * samplerate) # or pad to optimal FFT lengths - elif pad_to_length == 'nextpow2': + elif pad == 'nextpow2': abs_pad = _nextpow2(int(lenTrials.max())) # no padding in case of equal length trials - elif pad_to_length is None: + elif pad == 'maxperlen': abs_pad = int(lenTrials.max()) if lenTrials.min() != lenTrials.max(): msg = f"Unequal trial lengths present, padding all trials to {abs_pad} samples" - SPYWarning(msg) + SPYInfo(msg) # `abs_pad` is now the (soon to be padded) signal length in samples diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index 00566b0be..f81398f09 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -48,7 +48,7 @@ @detect_parallel_client def freqanalysis(data, method='mtmfft', output='pow', keeptrials=True, foi=None, foilim=None, - pad_to_length=None, polyremoval=None, taper="hann", + pad='maxperlen', polyremoval=None, taper="hann", taper_opt=None, tapsmofrq=None, nTaper=None, keeptapers=False, toi="all", t_ftimwin=None, wavelet="Morlet", width=6, order=None, order_max=None, order_min=1, c_1=3, adaptive=False, @@ -79,7 +79,7 @@ def freqanalysis(data, method='mtmfft', output='pow', * **tapsmofrq** : spectral smoothing box for slepian tapers (in Hz) * **nTaper** : number of orthogonal tapers for slepian tapers * **keeptapers** : return individual tapers or average - * **pad_to_length**: either pad to an absolute length or set to `'nextpow2'` + * **pad**: either pad to an absolute length or set to `'nextpow2'` "mtmconvol" : (Multi-)tapered sliding window Fourier transform Perform time-frequency analysis on time-series trial data based on a sliding @@ -145,15 +145,15 @@ def freqanalysis(data, method='mtmfft', output='pow', but may be unbounded (e.g., ``[-np.inf, 60.5]`` is valid). Edges `fmin` and `fmax` are included in the selection. If `foilim` is `None` or ``foilim = "all"``, all frequencies are selected. - pad_to_length : int, None or 'nextpow2' + pad : 'maxperlen', float or 'nextpow2' Padding of the input data, if set to a number pads all trials - to this absolute length. For instance ``pad_to_length = 2000`` pads all + to this absolute length in seconds. For instance ``pad=2`` pads all trials to an absolute length of 2000 samples, if and only if the longest - trial contains at maximum 2000 samples. - Alternatively `pad_to_length='nextpow2'` pads all trials to + trial contains at maximum 2000 samples and the samplerate is 1kHz. + Alternatively `pad='nextpow2'` pads all trials to the next power of two of the longest trial. - If `None` and trials have unequal lengths all trials are padded to match - the longest trial. + For the default `'maxperlen'` and trials have unequal lengths all + trials are padded to match the longest trial. polyremoval : int or None Order of polynomial used for de-trending data in the time domain prior to spectral analysis. A value of 0 corresponds to subtracting the mean @@ -334,18 +334,18 @@ def freqanalysis(data, method='mtmfft', output='pow', # --- Padding --- # Sliding window FFT does not support "fancy" padding - if method == "mtmconvol" and isinstance(pad_to_length, str): + if method == "mtmconvol" and isinstance(pad, str): msg = "method 'mtmconvol' only supports in-place padding for windows " +\ - "exceeding trial boundaries. Your choice of `pad_to_length = '{}'` will be ignored. " - SPYWarning(msg.format(pad_to_length)) + "exceeding trial boundaries. Your choice of `pad = '{}'` will be ignored. " + SPYWarning(msg.format(pad)) if method == 'mtmfft': # the actual number of samples in case of later padding - minSampleNum = process_padding(pad_to_length, lenTrials) + minSampleNum = process_padding(pad, lenTrials, data.samplerate) else: minSampleNum = lenTrials.min() - # Compute length (in samples) of shortest trial + # Compute length (in seconds) of shortest trial minTrialLength = minSampleNum / data.samplerate # Shortcut to data sampling interval @@ -367,7 +367,7 @@ def freqanalysis(data, method='mtmfft', output='pow', "keeptapers": keeptapers, "keeptrials": keeptrials, "polyremoval": polyremoval, - "pad_to_length": pad_to_length} + "pad": pad} # -------------------------------- # 1st: Check time-frequency inputs diff --git a/syncopy/tests/helpers.py b/syncopy/tests/helpers.py index bbea01c46..38f226de5 100644 --- a/syncopy/tests/helpers.py +++ b/syncopy/tests/helpers.py @@ -19,30 +19,30 @@ def run_padding_test(method_call, pad_length): """ The callable should test a solution and support - a single keyword argument `pad_to_length` + a single keyword argument `pad` """ - pad_options = [pad_length, 'nextpow2', None] + pad_options = [pad_length, 'nextpow2', 'maxperlen'] for pad in pad_options: - method_call(pad_to_length=pad) + method_call(pad=pad) # test invalid pads try: - method_call(pad_to_length=2) + method_call(pad=0.1) # trials should be longer than 0.1 seconds except SPYValueError as err: - assert 'pad_to_length' in str(err) + assert 'pad' in str(err) assert 'expected value to be greater' in str(err) try: - method_call(pad_to_length='IamNoPad') + method_call(pad='IamNoPad') except SPYValueError as err: - assert 'Invalid value of `pad_to_length`' in str(err) + assert 'Invalid value of `pad`' in str(err) assert 'nextpow2' in str(err) try: - method_call(pad_to_length=np.array([1000])) + method_call(pad=np.array([1000])) except SPYValueError as err: - assert 'Invalid value of `pad_to_length`' in str(err) + assert 'Invalid value of `pad`' in str(err) assert 'nextpow2' in str(err) diff --git a/syncopy/tests/test_connectivity.py b/syncopy/tests/test_connectivity.py index d0106e6c0..0a6bef483 100644 --- a/syncopy/tests/test_connectivity.py +++ b/syncopy/tests/test_connectivity.py @@ -147,8 +147,8 @@ def test_gr_parallel(self, testcluster=None): def test_gr_padding(self): - pad_length = int(1.7 * self.nSamples) - call = lambda pad_to_length: self.test_gr_solution(pad_to_length=pad_to_length) + pad_length = 6 # seconds + call = lambda pad: self.test_gr_solution(pad=pad) helpers.run_padding_test(call, pad_length) def test_gr_polyremoval(self): @@ -263,8 +263,8 @@ def test_coh_parallel(self, testcluster=None): def test_coh_padding(self): - pad_length = int(1.2 * self.nSamples) - call = lambda pad_to_length: self.test_coh_solution(pad_to_length=pad_to_length) + pad_length = 2 # seconds + call = lambda pad: self.test_coh_solution(pad=pad) helpers.run_padding_test(call, pad_length) def test_coh_polyremoval(self): @@ -354,25 +354,25 @@ def test_corr_solution(self, **kwargs): def test_corr_padding(self): - self.test_corr_solution(pad_to_length=None) + self.test_corr_solution(pad='maxperlen') # no padding is allowed for # this method try: - self.test_corr_solution(pad_to_length=1000) + self.test_corr_solution(pad=1000) except SPYValueError as err: - assert 'pad_to_length' in str(err) + assert 'pad' in str(err) assert 'no padding needed/allowed' in str(err) try: - self.test_corr_solution(pad_to_length='nextpow2') + self.test_corr_solution(pad='nextpow2') except SPYValueError as err: - assert 'pad_to_length' in str(err) + assert 'pad' in str(err) assert 'no padding needed/allowed' in str(err) try: - self.test_corr_solution(pad_to_length='IamNoPad') + self.test_corr_solution(pad='IamNoPad') except SPYValueError as err: - assert 'Invalid value of `pad_to_length`' in str(err) + assert 'Invalid value of `pad`' in str(err) assert 'no padding needed/allowed' in str(err) def test_corr_selections(self): diff --git a/syncopy/tests/test_specest.py b/syncopy/tests/test_specest.py index ecf2ce1d4..11f14223b 100644 --- a/syncopy/tests/test_specest.py +++ b/syncopy/tests/test_specest.py @@ -217,7 +217,7 @@ def test_solution(self): for sk, select in enumerate(self.sigdataSelections): sel = Selector(self.adata, select) spec = freqanalysis(self.adata, method="mtmfft", taper="hann", - pad_to_length="nextpow2", output="pow", select=select) + pad="nextpow2", output="pow", select=select) chanList = np.arange(self.nChannels)[sel.channel] amps = np.empty((len(sel.trials) * len(chanList),)) @@ -249,13 +249,13 @@ def test_foi(self): # offset `foi` by 0.1 Hz - resulting freqs must be unaffected ftmp = foi + 0.1 spec = freqanalysis(self.adata, method="mtmfft", taper="hann", - pad_to_length="nextpow2", foi=ftmp, select=select) + pad="nextpow2", foi=ftmp, select=select) assert np.all(spec.freq == foi) # unsorted, duplicate entries in `foi` - result must stay the same ftmp = np.hstack([foi, np.full(20, foi[0])]) spec = freqanalysis(self.adata, method="mtmfft", taper="hann", - pad_to_length="nextpow2", foi=ftmp, select=select) + pad="nextpow2", foi=ftmp, select=select) assert np.all(spec.freq == foi) def test_dpss(self): From 71de9601edeadf643bc0b8ec6ca44eee38c5bd3d Mon Sep 17 00:00:00 2001 From: tensionhead Date: Wed, 11 May 2022 15:53:24 +0200 Subject: [PATCH 32/38] CHG: Plotting tweaks - decreased legend size, removed legend frame and reverted to loc='best' - added check if figure got closed for CrossSpectralData plotting, such that a new one is created - threw in some tight_layout()s - corrected channel labeling for CrossSpectralData Changes to be committed: modified: syncopy/plotting/_plotting.py modified: syncopy/plotting/config.py modified: syncopy/plotting/sp_plotting.py --- syncopy/plotting/_plotting.py | 2 +- syncopy/plotting/config.py | 2 +- syncopy/plotting/sp_plotting.py | 37 ++++++++++++++++++++++++--------- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/syncopy/plotting/_plotting.py b/syncopy/plotting/_plotting.py index bd94a7ed8..3349ff4de 100644 --- a/syncopy/plotting/_plotting.py +++ b/syncopy/plotting/_plotting.py @@ -77,7 +77,7 @@ def plot_lines(ax, data_x, data_y, leg_fontsize=pltConfig['sLegendSize'], **pkwa else: ax.plot(data_x, data_y, **pkwargs) if 'label' in pkwargs: - ax.legend(ncol=2, loc='upper right', + ax.legend(ncol=2, loc='best', frameon=False, fontsize=leg_fontsize) # make room for the legend mn, mx = ax.get_ylim() diff --git a/syncopy/plotting/config.py b/syncopy/plotting/config.py index 4b1b6e012..2236da5df 100644 --- a/syncopy/plotting/config.py +++ b/syncopy/plotting/config.py @@ -22,7 +22,7 @@ "mTitleSize": 12.5, "mLabelSize": 12.5, "mTickSize": 11, - "mLegendSize": 11, + "mLegendSize": 10, "mXSize": 3.2, "mYSize": 2.4, "mMaxAxes": 35, diff --git a/syncopy/plotting/sp_plotting.py b/syncopy/plotting/sp_plotting.py index 0a4d330e5..3f3b1e548 100644 --- a/syncopy/plotting/sp_plotting.py +++ b/syncopy/plotting/sp_plotting.py @@ -56,7 +56,7 @@ def plot_AnalogData(data, shifted=True, **show_kwargs): fig, ax = _plotting.mk_line_figax() _plotting.plot_lines(ax, data_x, data_y, label=labels) - + fig.tight_layout() def plot_SpectralData(data, **show_kwargs): @@ -100,7 +100,8 @@ def plot_SpectralData(data, **show_kwargs): # need freq x time for plotting data_yx = data.show(**show_kwargs).T _plotting.plot_tfreq(ax, data_yx, time, data.freq) - ax.set_title(label, fontsize=pltConfig['sTitleSize']) + ax.set_title(label, fontsize=pltConfig['sTitleSize']) + fig.tight_layout() # just a line plot else: # get the data to plot @@ -114,7 +115,7 @@ def plot_SpectralData(data, **show_kwargs): ylabel='power (dB)') _plotting.plot_lines(ax, data_x, data_y, label=labels) - + fig.tight_layout() def plot_CrossSpectralData(data, **show_kwargs): """ @@ -141,26 +142,41 @@ def plot_CrossSpectralData(data, **show_kwargs): # what channel combination if 'channel_i' not in show_kwargs or 'channel_j' not in show_kwargs: - SPYWarning("Please select a channel combination for plotting!") + SPYWarning("Please select a channel combination `channel_i` and `channel_j` for plotting!") return chi, chj = show_kwargs['channel_i'], show_kwargs['channel_j'] + # parse labels + if isinstance(chi, str): + chi_label = chi + # must be int + else: + chi_label = f"channel{chi}" + # parse labels + if isinstance(chi, int): + chi_label = f"channel{chi + 1}" + else: + chi_label = chi + if isinstance(chj, int): + chj_label = f"channel{chj + 1}" + else: + chj_label = chj # what data do we have? method = plot_helpers.get_method(data) if method == 'granger': xlabel = 'frequency (Hz)' ylabel = 'Granger causality' - label = rf"channel{chi} $\rightarrow$ channel{chj}" + label = rf"{chi_label} $\rightarrow$ {chj_label}" data_x = plot_helpers.parse_foi(data, show_kwargs) elif method == 'coh': xlabel = 'frequency (Hz)' ylabel = 'coherence' - label = rf"channel{chi} - channel{chj}" + label = rf"{chi_label} - {chj_label}" data_x = plot_helpers.parse_foi(data, show_kwargs) elif method == 'corr': xlabel = 'lag' ylabel = 'correlation' - label = rf"channel{chi} - channel{chj}" + label = rf"{chi_label} - {chj_label}" data_x = plot_helpers.parse_toi(data, show_kwargs) # that's all the methods we got so far else: @@ -170,8 +186,9 @@ def plot_CrossSpectralData(data, **show_kwargs): data_y = data.show(**show_kwargs) # create the axes and figure if needed - # persisten axes allows for plotting different + # persistent axes allows for plotting different # channel combinations into the same figure - if not hasattr(data, 'ax'): - fig, data.ax = _plotting.mk_line_figax(xlabel, ylabel) + if not hasattr(data, 'fig') or not _plotting.ppl.fignum_exists(data.fig.number): + data.fig, data.ax = _plotting.mk_line_figax(xlabel, ylabel) _plotting.plot_lines(data.ax, data_x, data_y, label=label) + data.fig.tight_layout() From 878a8d3e852d5de1255ba91939bac858af266f5c Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 13 May 2022 10:18:46 +0200 Subject: [PATCH 33/38] CHG: Amended documentation of defintetrial - specifically mention how to create an "all-to-all" `trialdefinition` array by invoking `definetrial` without arguments - amended CHANGELOG On branch dev Changes to be committed: modified: CHANGELOG.md modified: syncopy/datatype/methods/definetrial.py --- CHANGELOG.md | 6 ++++-- syncopy/datatype/methods/definetrial.py | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad0c27042..6f7d7678e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - ## [Unreleased] -### NEW +Bugfixes and features additions for `EventData` objects. + ### NEW - Added support for flexible columns in `EventData` (thanks to @KatharineShapcott) ### CHANGED +- Include specific example how to create an "all-to-all" `trialdefinition` array + by invoking `definetrial` without arguments in the function's docstring. ### REMOVED diff --git a/syncopy/datatype/methods/definetrial.py b/syncopy/datatype/methods/definetrial.py index fa5489790..5a9a7acb2 100644 --- a/syncopy/datatype/methods/definetrial.py +++ b/syncopy/datatype/methods/definetrial.py @@ -24,6 +24,9 @@ def definetrial(obj, trialdefinition=None, pre=None, post=None, start=None, start trigger stop |---- pre ----|--------|---------|--- post----| + **Note**: To define a trial encompassing the whole dataset simply invoke this + routine with no arguments, i.e., ``definetrial(obj)`` or equivalently + ``obj.definetrial()`` Parameters ---------- From 5cd0d7e1b597e9ef86d2c5bd80dfa6d51aac6ae9 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 13 May 2022 11:10:10 +0200 Subject: [PATCH 34/38] CHG: Typos and phrasing - fixed docstrings in `connectivityanalysis` On branch FT-padding Changes to be committed: modified: syncopy/nwanalysis/connectivity_analysis.py modified: syncopy/shared/input_processors.py modified: syncopy/specest/freqanalysis.py --- syncopy/nwanalysis/connectivity_analysis.py | 16 ++++++++-------- syncopy/shared/input_processors.py | 6 +++--- syncopy/specest/freqanalysis.py | 16 ++++++++-------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/syncopy/nwanalysis/connectivity_analysis.py b/syncopy/nwanalysis/connectivity_analysis.py index fe7df9abb..87da913fe 100644 --- a/syncopy/nwanalysis/connectivity_analysis.py +++ b/syncopy/nwanalysis/connectivity_analysis.py @@ -102,14 +102,14 @@ def connectivityanalysis(data, method="coh", keeptrials=False, output="abs", `foi` array will be constructed in 1Hz steps from `fmin` to `fmax` (inclusive). pad : 'maxperlen', float or 'nextpow2' - Padding of the input data, if set to a number pads all trials - to this absolute length in seconds. For instance ``pad=2`` pads all - trials to an absolute length of 2000 samples, if and only if the longest - trial contains at maximum 2000 samples and the samplerate is 1kHz. - Alternatively `pad='nextpow2'` pads all trials to - the next power of two of the longest trial. - For the default `'maxperlen'` and trials have unequal lengths all - trials are padded to match the longest trial. + For the default `maxperlen`, no padding is performed in case of equal + length trials, while trials of varying lengths are padded to match the + longest trial. If `pad` is a number all trials are padded so that `pad` indicates + the absolute length of all trials after padding (in seconds). For instance + ``pad = 2`` pads all trials to an absolute length of 2000 samples, if and + only if the longest trial contains at maximum 2000 samples and the + samplerate is 1kHz. If `pad` is `'nextpow2'` all trials are padded to the + nearest power of two (in samples) of the longest trial. tapsmofrq : float or None Only valid if `method` is `'coh'` or `'granger'`. Enables multi-tapering and sets the amount of spectral diff --git a/syncopy/shared/input_processors.py b/syncopy/shared/input_processors.py index 8b92fb7e0..dd7fc021b 100644 --- a/syncopy/shared/input_processors.py +++ b/syncopy/shared/input_processors.py @@ -26,7 +26,7 @@ def process_padding(pad, lenTrials, samplerate): padding has to be done **after** tapering! This function returns a number indicating the total - length in #samples of all trials after padding. + length in sample-count of all trials after padding. Parameters ---------- @@ -36,7 +36,7 @@ def process_padding(pad, lenTrials, samplerate): trials get padded to the max. trial length. A float indicates the absolute length of all trials after padding in seconds. `'nextpow2'` pads all trials - to the nearest power of two. + to the nearest power of two. lenTrials : sequence of int_like Sequence holding all individual trial lengths samplerate : float @@ -45,7 +45,7 @@ def process_padding(pad, lenTrials, samplerate): Returns ------- abs_pad : int - Absolute length of all trials after padding in #samples + Absolute length of all trials after padding (in samples) """ # supported padding options diff --git a/syncopy/specest/freqanalysis.py b/syncopy/specest/freqanalysis.py index f81398f09..1379c9986 100644 --- a/syncopy/specest/freqanalysis.py +++ b/syncopy/specest/freqanalysis.py @@ -146,14 +146,14 @@ def freqanalysis(data, method='mtmfft', output='pow', and `fmax` are included in the selection. If `foilim` is `None` or ``foilim = "all"``, all frequencies are selected. pad : 'maxperlen', float or 'nextpow2' - Padding of the input data, if set to a number pads all trials - to this absolute length in seconds. For instance ``pad=2`` pads all - trials to an absolute length of 2000 samples, if and only if the longest - trial contains at maximum 2000 samples and the samplerate is 1kHz. - Alternatively `pad='nextpow2'` pads all trials to - the next power of two of the longest trial. - For the default `'maxperlen'` and trials have unequal lengths all - trials are padded to match the longest trial. + For the default `maxperlen`, no padding is performed in case of equal + length trials, while trials of varying lengths are padded to match the + longest trial. If `pad` is a number all trials are padded so that `pad` indicates + the absolute length of all trials after padding (in seconds). For instance + ``pad = 2`` pads all trials to an absolute length of 2000 samples, if and + only if the longest trial contains at maximum 2000 samples and the + samplerate is 1kHz. If `pad` is `'nextpow2'` all trials are padded to the + nearest power of two (in samples) of the longest trial. polyremoval : int or None Order of polynomial used for de-trending data in the time domain prior to spectral analysis. A value of 0 corresponds to subtracting the mean From a97e0a02c073c8563bf3af987ec23336f340a84e Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 13 May 2022 11:37:39 +0200 Subject: [PATCH 35/38] CHG: Modified padding test - use negative `pad` value to trigger exception On branch FT-padding Changes to be committed: modified: syncopy/tests/helpers.py --- syncopy/tests/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncopy/tests/helpers.py b/syncopy/tests/helpers.py index 38f226de5..ea8694cff 100644 --- a/syncopy/tests/helpers.py +++ b/syncopy/tests/helpers.py @@ -28,7 +28,7 @@ def run_padding_test(method_call, pad_length): # test invalid pads try: - method_call(pad=0.1) # trials should be longer than 0.1 seconds + method_call(pad=-0.1) # trials should be longer than 0.1 seconds except SPYValueError as err: assert 'pad' in str(err) assert 'expected value to be greater' in str(err) From 6c82d431513b7bb212926d63dc6d3e22842950b7 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 13 May 2022 11:43:18 +0200 Subject: [PATCH 36/38] CHG: Updated CHANGELOG - updated CHANGELOG in preparation for upcoming release On branch dev Changes to be committed: modified: CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f7d7678e..9eb2ccbed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ Bugfixes and features additions for `EventData` objects. ### CHANGED - Include specific example how to create an "all-to-all" `trialdefinition` array by invoking `definetrial` without arguments in the function's docstring. +- Modified versioning scheme: use a date-based scheme instead of increasing + version numbers +- Aligned padding API to FieldTrip in both `freqanalysis` and `connectivityanalysis`: + use `pad` instead of `pad_to_length` with three supported modes ('maxperlen', + float, 'nextpow2'). ### REMOVED From 79aced904df582179538e4a06e9c76e8551e2e8e Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 13 May 2022 12:12:54 +0200 Subject: [PATCH 37/38] FIX: Repaired SLURM tests - fixed working dir of SLURM tests so that custom `--full` options gets picked up correctly On branch dev Changes to be committed: modified: .gitlab-ci.yml --- .gitlab-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c92fcdfe8..4b7e82395 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -99,10 +99,11 @@ slurmtest: - conda env update -f syncopy.yml --prune - conda activate syncopy - export PYTHONPATH=$CI_PROJECT_DIR - - srun -p DEV --mem=8000m -c 4 pytest --full $TEST_DIR/test_specest.py -k 'not para' - - srun -p DEV --mem=8000m -c 4 pytest --full $TEST_DIR/test_specest.py -k 'para' - - srun -p DEV --mem=8000m -c 4 pytest --full $TEST_DIR/test_connectivity.py - - srun -p DEV --mem=8000m -c 4 pytest --full --ignore=$TEST_DIR/test_specest.py --ignore=$TEST_DIR/test_connectivity.py + - cd $TEST_DIR + - srun -p DEV --mem=8000m -c 4 pytest --full test_specest.py -k 'not para' + - srun -p DEV --mem=8000m -c 4 pytest --full test_specest.py -k 'para' + - srun -p DEV --mem=8000m -c 4 pytest --full test_connectivity.py + - srun -p DEV --mem=8000m -c 4 pytest --full --ignore=test_specest.py --ignore=test_connectivity.py pypitest: stage: upload From dd324966b3948493dee5015b1118c8260aaba2b3 Mon Sep 17 00:00:00 2001 From: Stefan Fuertinger Date: Fri, 13 May 2022 12:14:49 +0200 Subject: [PATCH 38/38] CHG: Updated version + CHANGELOG - preps for new release On branch dev Changes to be committed: modified: CHANGELOG.md modified: setup.py --- CHANGELOG.md | 4 +--- setup.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eb2ccbed..2d30368c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [2022.05] - 2022-05-13 Bugfixes and features additions for `EventData` objects. ### NEW @@ -19,8 +19,6 @@ Bugfixes and features additions for `EventData` objects. use `pad` instead of `pad_to_length` with three supported modes ('maxperlen', float, 'nextpow2'). -### REMOVED - ### DEPRECATED - Removed support for calling `freqanalysis` with a `toi` array as well as an input dataset that has an active in-place time-selection attached diff --git a/setup.py b/setup.py index 661872749..b977a9fb9 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ from conda2pip import conda2pip # Set release version by hand for master branch -releaseVersion = "0.22" +releaseVersion = "2022.05" # Get necessary and optional package dependencies required, dev = conda2pip(return_lists=True)