diff --git a/CHANGES.rst b/CHANGES.rst index 5e18f09..cd5e82b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,20 @@ Changelog ========= +.. _changes-1_6_0: + +1.6.0 (2024-12-06) +~~~~~~~~~~~~~~~~~~ + +New features +------------ + ++ `#166`_: Add new custom IDS method + :meth:`icat.client.Client.restoreData`. + +.. _#166: https://github.com/icatproject/python-icat/pull/166 + + .. _changes-1_5_1: 1.5.1 (2024-10-25) diff --git a/doc/src/client.rst b/doc/src/client.rst index 051edac..31aa808 100644 --- a/doc/src/client.rst +++ b/doc/src/client.rst @@ -154,4 +154,6 @@ manages the interaction with an ICAT service as a client. .. automethod:: deleteData + .. automethod:: restoreData + .. _ICAT SOAP Manual: https://repo.icatproject.org/site/icat/server/4.10.0/soap.html diff --git a/src/icat/client.py b/src/icat/client.py index 8c0cf2c..8377515 100644 --- a/src/icat/client.py +++ b/src/icat/client.py @@ -1000,5 +1000,40 @@ def deleteData(self, objs): objs = DataSelection(objs) self.ids.delete(objs) + def restoreData(self, objs): + """Request IDS to restore data. + + Check the status of the data, request a restore if needed and + wait for the restore to complete. + + :param objs: either a dict having some of the keys + `investigationIds`, `datasetIds`, and `datafileIds` + with a list of object ids as value respectively, or a list + of entity objects, or a data selection. + :type objs: :class:`dict`, :class:`list` of + :class:`icat.entity.Entity`, or + :class:`icat.ids.DataSelection` + + .. versionadded:: 1.6.0 + """ + if not self.ids: + raise RuntimeError("no IDS.") + if not isinstance(objs, DataSelection): + objs = DataSelection(objs) + while True: + self.autoRefresh() + status = self.ids.getStatus(objs) + if status == "ONLINE": + break + elif status == "RESTORING": + pass + elif status == "ARCHIVED": + self.ids.restore(objs) + else: + # Should never happen + raise IDSResponseError("unexpected response from " + "IDS getStatus() call: %s" % status) + time.sleep(30) + atexit.register(Client.cleanupall) diff --git a/tests/conftest.py b/tests/conftest.py index 104901d..9c5a8cf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -48,9 +48,28 @@ logging.getLogger('suds.client').setLevel(logging.CRITICAL) logging.getLogger('suds').setLevel(logging.ERROR) +_skip_slow = True testdir = Path(__file__).resolve().parent testdatadir = testdir / "data" +def pytest_addoption(parser): + parser.addoption("--no-skip-slow", action="store_true", default=False, + help="do not skip slow tests.") + +def pytest_configure(config): + global _skip_slow + _skip_slow = not config.getoption("--no-skip-slow") + config.addinivalue_line("markers", "slow: mark a test as slow, " + "the test will be skipped unless --no-skip-slow " + "is set on the command line") + +def pytest_runtest_setup(item): + """Skip slow tests by default. + """ + marker = item.get_closest_marker("slow") + if marker is not None and _skip_slow: + pytest.skip("skip slow test") + def _skip(reason): if Version(pytest.__version__) >= '3.3.0': pytest.skip(reason, allow_module_level=True) diff --git a/tests/test_06_ids.py b/tests/test_06_ids.py index d9c42ed..3a4fc3d 100644 --- a/tests/test_06_ids.py +++ b/tests/test_06_ids.py @@ -3,23 +3,31 @@ import datetime import filecmp +import logging import time import zipfile import pytest import icat +import icat.client import icat.config -from icat.ids import DataSelection +from icat.ids import IDSClient, DataSelection from icat.query import Query from conftest import DummyDatafile, UtcTimezone from conftest import getConfig, tmpSessionId, tmpClient +logger = logging.getLogger(__name__) -@pytest.fixture(scope="module") -def client(setupicat): - client, conf = getConfig(ids="mandatory") - client.login(conf.auth, conf.credentials) - yield client - query = "SELECT df FROM Datafile df WHERE df.location IS NOT NULL" +GiB = 1073741824 + +class LoggingIDSClient(IDSClient): + """Modified version of IDSClient that logs some calls. + """ + def getStatus(self, selection): + status = super().getStatus(selection) + logger.debug("getStatus(%s): %s", selection, status, stacklevel=2) + return status + +def _delete_datafiles(client, query): while True: try: client.deleteData(client.search(query)) @@ -28,6 +36,45 @@ def client(setupicat): else: break +@pytest.fixture(scope="module") +def cleanup(setupicat): + client, conf = getConfig(confSection="root", ids="mandatory") + client.login(conf.auth, conf.credentials) + yield + query = "SELECT df FROM Datafile df WHERE df.location IS NOT NULL" + _delete_datafiles(client, query) + +@pytest.fixture(scope="function") +def client(monkeypatch, cleanup): + monkeypatch.setattr(icat.client, "IDSClient", LoggingIDSClient) + client, conf = getConfig(ids="mandatory") + client.login(conf.auth, conf.credentials) + yield client + +@pytest.fixture(scope="function") +def dataset(client, cleanup_objs): + """A dataset to be used in the test. + + The dataset will be eventually be deleted after the test. + """ + inv = client.assertedSearch(Query(client, "Investigation", conditions={ + "name": "= '10100601-ST'", + }))[0] + dstype = client.assertedSearch(Query(client, "DatasetType", conditions={ + "name": "= 'raw'", + }))[0] + dataset = client.new("Dataset", + name="e208343", complete=False, + investigation=inv, type=dstype) + dataset.create() + cleanup_objs.append(dataset) + yield dataset + query = Query(client, "Datafile", conditions={ + "dataset.id": "= %d" % dataset.id, + "location": "IS NOT NULL", + }) + _delete_datafiles(client, query) + # ============================ testdata ============================ @@ -392,6 +439,55 @@ def test_restore(client, case): # outcome of the restore() call. print("Status of dataset %s is now %s" % (case['dsname'], status)) +@pytest.mark.parametrize(("case"), markeddatasets) +def test_restoreDataCall(client, case): + """Test the high level call restoreData(). + + This is essentially a no-op as the dataset in question will + already be ONLINE. It only tests that the call does not throw an + error. + """ + dataset = getDataset(client, case) + client.restoreData([dataset]) + status = client.ids.getStatus(DataSelection([dataset])) + assert status == "ONLINE" + +@pytest.mark.parametrize(("case"), markeddatasets) +def test_restoreDataCallSelection(client, case): + """Test the high level call restoreData(). + + Same as last test, but now pass a DataSelection as argument. + """ + selection = DataSelection([getDataset(client, case)]) + client.restoreData(selection) + status = client.ids.getStatus(selection) + assert status == "ONLINE" + +@pytest.mark.slow +def test_restoreData(tmpdirsec, client, dataset): + """Test restoring data with the high level call restoreData(). + + This test archives a dataset and calls restoreData() to restore it + again. The size of the dataset is large enough so that restoring + takes some time, so that we actually can observe the call to wait + until the restoring is finished. As a result, the test is rather + slow. It is marked as such and thus disabled by default. + """ + if not client.ids.isTwoLevel(): + pytest.skip("This IDS does not use two levels of data storage") + f = DummyDatafile(tmpdirsec, "e208343.nxs", GiB) + query = Query(client, "DatafileFormat", conditions={ + "name": "= 'NeXus'", + }) + datafileformat = client.assertedSearch(query)[0] + datafile = client.new("Datafile", name=f.fname.name, + dataset=dataset, datafileFormat=datafileformat) + client.putData(f.fname, datafile) + client.ids.archive(DataSelection([dataset])) + client.restoreData([dataset]) + status = client.ids.getStatus(DataSelection([dataset])) + assert status == "ONLINE" + @pytest.mark.parametrize(("case"), markeddatasets) def test_reset(client, case): """Call reset() on a dataset.