diff --git a/docs/python.md b/docs/python.md index cb7f909..844d9c7 100644 --- a/docs/python.md +++ b/docs/python.md @@ -214,6 +214,24 @@ pip install "ngff-zarr[tensorstore]" nz.to_ngff_zarr('cthead1.ome.zarr', multiscales, use_tensorstore=True) ``` +## Convert OME-Zarr versions + +To convert from OME-Zarr version 0.4, which uses the Zarr Format Specification +2, to 0.5, which uses the Zarr Format Specification 3, or vice version, specify +the desired version when writing. + +```python +# Convert from 0.4 to 0.5 +multiscales = from_ngff_zarr('cthead1.ome.zarr') +to_ngff_zarr('cthead1_zarr3.ome.zarr', multiscales, version='0.5') +``` + +```python +# Convert from 0.5 to 0.4 +multiscales = from_ngff_zarr('cthead1.ome.zarr') +to_ngff_zarr('cthead1_zarr2.ome.zarr', multiscales, version='0.4') +``` + [dataclass]: https://docs.python.org/3/library/dataclasses.html [dataclasses]: https://docs.python.org/3/library/dataclasses.html [Dask arrays]: https://docs.dask.org/en/stable/array.html diff --git a/ngff_zarr/from_ngff_zarr.py b/ngff_zarr/from_ngff_zarr.py index a937fc1..2c8bc85 100644 --- a/ngff_zarr/from_ngff_zarr.py +++ b/ngff_zarr/from_ngff_zarr.py @@ -2,6 +2,7 @@ from pathlib import Path from typing import Union, Optional import packaging.version +import sys import dask.array import zarr @@ -91,6 +92,11 @@ def from_ngff_zarr( datasets = [] for dataset in metadata["datasets"]: data = dask.array.from_zarr(store, component=dataset["path"]) + # Convert endianness to native if needed + if (sys.byteorder == "little" and data.dtype.byteorder == ">") or ( + sys.byteorder == "big" and data.dtype.byteorder == "<" + ): + data = data.astype(data.dtype.newbyteorder()) scale = {d: 1.0 for d in dims} translation = {d: 0.0 for d in dims} diff --git a/ngff_zarr/to_ngff_zarr.py b/ngff_zarr/to_ngff_zarr.py index 69962b1..80a5d2b 100644 --- a/ngff_zarr/to_ngff_zarr.py +++ b/ngff_zarr/to_ngff_zarr.py @@ -17,6 +17,8 @@ import zarr import zarr.storage from ._zarr_open_array import open_array +from .v04.zarr_metadata import Metadata as Metadata_v04 +from .v05.zarr_metadata import Metadata as Metadata_v05 # Zarr Python 3 if hasattr(zarr.storage, "StoreLike"): @@ -182,7 +184,23 @@ def to_ngff_zarr( if version != "0.4" and version != "0.5": raise ValueError(f"Unsupported version: {version}") - metadata_dict = asdict(multiscales.metadata) + metadata = multiscales.metadata + if version == "0.4" and isinstance(metadata, Metadata_v05): + metadata = Metadata_v04( + axes=metadata.axes, + datasets=metadata.datasets, + coordinateTransformations=metadata.coordinateTransformations, + name=metadata.name, + ) + if version == "0.5" and isinstance(metadata, Metadata_v04): + metadata = Metadata_v05( + axes=metadata.axes, + datasets=metadata.datasets, + coordinateTransformations=metadata.coordinateTransformations, + name=metadata.name, + ) + + metadata_dict = asdict(metadata) metadata_dict = _pop_metadata_optionals(metadata_dict) metadata_dict["@type"] = "ngff:Image" zarr_format = 2 if version == "0.4" else 3 @@ -224,7 +242,7 @@ def to_ngff_zarr( progress.update_multiscales_task_completed(index + 1) image = next_image arr = image.data - path = multiscales.metadata.datasets[index].path + path = metadata.datasets[index].path parent = str(PurePosixPath(path).parent) if parent not in (".", "/"): array_dims_group = root.create_group(parent) diff --git a/ngff_zarr/validate.py b/ngff_zarr/validate.py index da46bbe..96a4cc5 100644 --- a/ngff_zarr/validate.py +++ b/ngff_zarr/validate.py @@ -1,6 +1,7 @@ from typing import Dict from pathlib import Path import json +from packaging import version as packaging_version from importlib_resources import files as file_resources @@ -40,5 +41,10 @@ def validate( registry = Registry().with_resource( NGFF_URI, resource=Resource.from_contents(schema) ) + if packaging_version.parse(version) >= packaging_version.parse("0.5"): + version_schema = load_schema(version=version, model="_version") + registry = registry.with_resource( + NGFF_URI, resource=Resource.from_contents(version_schema) + ) validator = Draft202012Validator(schema, registry=registry) validator.validate(ngff_dict) diff --git a/pixi.lock b/pixi.lock index dc2693e..f65411b 100644 --- a/pixi.lock +++ b/pixi.lock @@ -2140,7 +2140,7 @@ packages: version: 0.4.6 url: https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl sha256: 4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 - requires_python: '!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7' + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*' - kind: pypi name: crc32c version: 2.7.1 @@ -2191,7 +2191,7 @@ packages: - lz4>=4.3.2 ; extra == 'complete' - dask[array] ; extra == 'dataframe' - pandas>=2.0 ; extra == 'dataframe' - - dask-expr<1.2,>=1.1 ; extra == 'dataframe' + - dask-expr>=1.1,<1.2 ; extra == 'dataframe' - bokeh>=3.1.0 ; extra == 'diagnostics' - jinja2>=2.10.3 ; extra == 'diagnostics' - distributed==2024.11.2 ; extra == 'distributed' @@ -2388,10 +2388,10 @@ packages: - pytest-recording ; extra == 'test' - pytest-rerunfailures ; extra == 'test' - requests ; extra == 'test' - - aiobotocore<3.0.0,>=2.5.4 ; extra == 'test-downstream' + - aiobotocore>=2.5.4,<3.0.0 ; extra == 'test-downstream' - dask-expr ; extra == 'test-downstream' - dask[dataframe,test] ; extra == 'test-downstream' - - moto[server]<5,>4 ; extra == 'test-downstream' + - moto[server]>4,<5 ; extra == 'test-downstream' - pytest-timeout ; extra == 'test-downstream' - xarray ; extra == 'test-downstream' - adlfs ; extra == 'test-full' @@ -2441,7 +2441,7 @@ packages: sha256: 6cd97c58b47813d3619e63e9081169880fbe331f0ca883c871ff1f3f11814f5c requires_dist: - beautifulsoup4 - - sphinx<9.0,>=6.0 + - sphinx>=6.0,<9.0 - sphinx-basic-ng>=1.0.0b2 - pygments>=2.7 requires_python: '>=3.8' @@ -2754,7 +2754,7 @@ packages: - sphinx-lint ; extra == 'doc' - jaraco-tidelift>=1.4 ; extra == 'doc' - pytest-enabler>=2.2 ; extra == 'enabler' - - pytest!=8.1.*,>=6 ; extra == 'test' + - pytest>=6,!=8.1.* ; extra == 'test' - zipp>=3.17 ; extra == 'test' - jaraco-test>=5.4 ; extra == 'test' - pytest-mypy ; extra == 'type' @@ -4003,22 +4003,22 @@ packages: - dask-image ; extra == 'all' - dask[distributed] ; extra == 'all' - deepdiff ; extra == 'all' - - furo<2025,>=2024.7.18 ; extra == 'all' + - furo>=2024.7.18,<2025 ; extra == 'all' - imagecodecs ; extra == 'all' - imageio ; extra == 'all' - itk-filtering>=5.3.0 ; extra == 'all' - itk-io>=5.3.0 ; extra == 'all' - itkwasm-image-io ; extra == 'all' - jsonschema ; extra == 'all' - - myst-parser<4,>=3.0.1 ; extra == 'all' + - myst-parser>=3.0.1,<4 ; extra == 'all' - pooch ; extra == 'all' - pytest>=6 ; extra == 'all' - - sphinx-autobuild<2025,>=2024.4.16 ; extra == 'all' - - sphinx-autodoc2<0.6,>=0.5.0 ; extra == 'all' - - sphinx-copybutton<0.6,>=0.5.2 ; extra == 'all' - - sphinx-design<0.7,>=0.6.0 ; extra == 'all' - - sphinx<8,>=7.4.7 ; extra == 'all' - - sphinxext-opengraph<0.10,>=0.9.1 ; extra == 'all' + - sphinx-autobuild>=2024.4.16,<2025 ; extra == 'all' + - sphinx-autodoc2>=0.5.0,<0.6 ; extra == 'all' + - sphinx-copybutton>=0.5.2,<0.6 ; extra == 'all' + - sphinx-design>=0.6.0,<0.7 ; extra == 'all' + - sphinx>=7.4.7,<8 ; extra == 'all' + - sphinxext-opengraph>=0.9.1,<0.10 ; extra == 'all' - tensorstore ; extra == 'all' - tifffile>=2024.7.24 ; extra == 'all' - dask-image ; extra == 'cli' @@ -4030,14 +4030,14 @@ packages: - itkwasm-image-io ; extra == 'cli' - tifffile>=2024.7.24 ; extra == 'cli' - dask-image ; extra == 'dask-image' - - furo<2025,>=2024.7.18 ; extra == 'docs' - - myst-parser<4,>=3.0.1 ; extra == 'docs' - - sphinx-autobuild<2025,>=2024.4.16 ; extra == 'docs' - - sphinx-autodoc2<0.6,>=0.5.0 ; extra == 'docs' - - sphinx-copybutton<0.6,>=0.5.2 ; extra == 'docs' - - sphinx-design<0.7,>=0.6.0 ; extra == 'docs' - - sphinx<8,>=7.4.7 ; extra == 'docs' - - sphinxext-opengraph<0.10,>=0.9.1 ; extra == 'docs' + - furo>=2024.7.18,<2025 ; extra == 'docs' + - myst-parser>=3.0.1,<4 ; extra == 'docs' + - sphinx-autobuild>=2024.4.16,<2025 ; extra == 'docs' + - sphinx-autodoc2>=0.5.0,<0.6 ; extra == 'docs' + - sphinx-copybutton>=0.5.2,<0.6 ; extra == 'docs' + - sphinx-design>=0.6.0,<0.7 ; extra == 'docs' + - sphinx>=7.4.7,<8 ; extra == 'docs' + - sphinxext-opengraph>=0.9.1,<0.10 ; extra == 'docs' - itk-filtering>=5.3.0 ; extra == 'itk' - tensorstore ; extra == 'tensorstore' - deepdiff ; extra == 'test' @@ -4077,7 +4077,7 @@ packages: - pydata-sphinx-theme ; extra == 'docs' - numpydoc ; extra == 'docs' - msgpack ; extra == 'msgpack' - - pcodec<0.3,>=0.2 ; extra == 'pcodec' + - pcodec>=0.2,<0.3 ; extra == 'pcodec' - coverage ; extra == 'test' - pytest ; extra == 'test' - pytest-cov ; extra == 'test' @@ -4103,7 +4103,7 @@ packages: - msgpack ; extra == 'msgpack' - zfpy>=1.0.0 ; extra == 'zfpy' - numpy<2.0.0 ; extra == 'zfpy' - - pcodec<0.3,>=0.2 ; extra == 'pcodec' + - pcodec>=0.2,<0.3 ; extra == 'pcodec' - crc32c>=2.7 ; extra == 'crc32c' requires_python: '>=3.11' - kind: pypi @@ -4124,7 +4124,7 @@ packages: - msgpack ; extra == 'msgpack' - zfpy>=1.0.0 ; extra == 'zfpy' - numpy<2.0.0 ; extra == 'zfpy' - - pcodec<0.3,>=0.2 ; extra == 'pcodec' + - pcodec>=0.2,<0.3 ; extra == 'pcodec' - crc32c>=2.7 ; extra == 'crc32c' requires_python: '>=3.11' - kind: pypi @@ -4145,7 +4145,7 @@ packages: - msgpack ; extra == 'msgpack' - zfpy>=1.0.0 ; extra == 'zfpy' - numpy<2.0.0 ; extra == 'zfpy' - - pcodec<0.3,>=0.2 ; extra == 'pcodec' + - pcodec>=0.2,<0.3 ; extra == 'pcodec' - crc32c>=2.7 ; extra == 'crc32c' requires_python: '>=3.11' - kind: pypi @@ -4166,7 +4166,7 @@ packages: - msgpack ; extra == 'msgpack' - zfpy>=1.0.0 ; extra == 'zfpy' - numpy<2.0.0 ; extra == 'zfpy' - - pcodec<0.3,>=0.2 ; extra == 'pcodec' + - pcodec>=0.2,<0.3 ; extra == 'pcodec' - crc32c>=2.7 ; extra == 'crc32c' requires_python: '>=3.11' - kind: pypi @@ -5268,7 +5268,7 @@ packages: requires_dist: - iniconfig - packaging - - pluggy<2,>=1.5 + - pluggy>=1.5,<2 - exceptiongroup>=1.0.0rc8 ; python_full_version < '3.11' - tomli>=1 ; python_full_version < '3.11' - colorama ; sys_platform == 'win32' @@ -5433,7 +5433,7 @@ packages: sha256: a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 requires_dist: - six>=1.5 - requires_python: '!=3.0.*,!=3.1.*,!=3.2.*,>=2.7' + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*' - kind: conda name: python_abi version: '3.13' @@ -5777,7 +5777,7 @@ packages: url: https://files.pythonhosted.org/packages/50/ef/ac98346db016ff18a6ad7626a35808f37074d25796fd0234c2bb0ed1e054/scipy-1.14.1-cp313-cp313-macosx_10_13_x86_64.whl sha256: 1729560c906963fc8389f6aac023739ff3983e727b1a4d87696b7bf108316a79 requires_dist: - - numpy<2.3,>=1.23.5 + - numpy>=1.23.5,<2.3 - pytest ; extra == 'test' - pytest-cov ; extra == 'test' - pytest-timeout ; extra == 'test' @@ -5793,7 +5793,7 @@ packages: - cython ; extra == 'test' - meson ; extra == 'test' - ninja ; sys_platform != 'emscripten' and extra == 'test' - - sphinx<=7.3.7,>=5.0.0 ; extra == 'doc' + - sphinx>=5.0.0,<=7.3.7 ; extra == 'doc' - pydata-sphinx-theme>=0.15.2 ; extra == 'doc' - sphinx-design>=0.4.0 ; extra == 'doc' - matplotlib>=3.5 ; extra == 'doc' @@ -5819,7 +5819,7 @@ packages: url: https://files.pythonhosted.org/packages/56/46/2449e6e51e0d7c3575f289f6acb7f828938eaab8874dbccfeb0cd2b71a27/scipy-1.14.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl sha256: 5149e3fd2d686e42144a093b206aef01932a0059c2a33ddfa67f5f035bdfe13e requires_dist: - - numpy<2.3,>=1.23.5 + - numpy>=1.23.5,<2.3 - pytest ; extra == 'test' - pytest-cov ; extra == 'test' - pytest-timeout ; extra == 'test' @@ -5835,7 +5835,7 @@ packages: - cython ; extra == 'test' - meson ; extra == 'test' - ninja ; sys_platform != 'emscripten' and extra == 'test' - - sphinx<=7.3.7,>=5.0.0 ; extra == 'doc' + - sphinx>=5.0.0,<=7.3.7 ; extra == 'doc' - pydata-sphinx-theme>=0.15.2 ; extra == 'doc' - sphinx-design>=0.4.0 ; extra == 'doc' - matplotlib>=3.5 ; extra == 'doc' @@ -5861,7 +5861,7 @@ packages: url: https://files.pythonhosted.org/packages/a7/2f/6c142b352ac15967744d62b165537a965e95d557085db4beab2a11f7943b/scipy-1.14.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl sha256: b99722ea48b7ea25e8e015e8341ae74624f72e5f21fc2abd45f3a93266de4c5d requires_dist: - - numpy<2.3,>=1.23.5 + - numpy>=1.23.5,<2.3 - pytest ; extra == 'test' - pytest-cov ; extra == 'test' - pytest-timeout ; extra == 'test' @@ -5877,7 +5877,7 @@ packages: - cython ; extra == 'test' - meson ; extra == 'test' - ninja ; sys_platform != 'emscripten' and extra == 'test' - - sphinx<=7.3.7,>=5.0.0 ; extra == 'doc' + - sphinx>=5.0.0,<=7.3.7 ; extra == 'doc' - pydata-sphinx-theme>=0.15.2 ; extra == 'doc' - sphinx-design>=0.4.0 ; extra == 'doc' - matplotlib>=3.5 ; extra == 'doc' @@ -5903,7 +5903,7 @@ packages: url: https://files.pythonhosted.org/packages/b9/cc/70948fe9f393b911b4251e96b55bbdeaa8cca41f37c26fd1df0232933b9e/scipy-1.14.1-cp313-cp313-macosx_12_0_arm64.whl sha256: 4079b90df244709e675cdc8b93bfd8a395d59af40b72e339c2287c91860deb8e requires_dist: - - numpy<2.3,>=1.23.5 + - numpy>=1.23.5,<2.3 - pytest ; extra == 'test' - pytest-cov ; extra == 'test' - pytest-timeout ; extra == 'test' @@ -5919,7 +5919,7 @@ packages: - cython ; extra == 'test' - meson ; extra == 'test' - ninja ; sys_platform != 'emscripten' and extra == 'test' - - sphinx<=7.3.7,>=5.0.0 ; extra == 'doc' + - sphinx>=5.0.0,<=7.3.7 ; extra == 'doc' - pydata-sphinx-theme>=0.15.2 ; extra == 'doc' - sphinx-design>=0.4.0 ; extra == 'doc' - matplotlib>=3.5 ; extra == 'doc' @@ -5945,7 +5945,7 @@ packages: url: https://files.pythonhosted.org/packages/f5/1b/6ee032251bf4cdb0cc50059374e86a9f076308c1512b61c4e003e241efb7/scipy-1.14.1-cp313-cp313-win_amd64.whl sha256: baff393942b550823bfce952bb62270ee17504d02a1801d7fd0719534dfb9c84 requires_dist: - - numpy<2.3,>=1.23.5 + - numpy>=1.23.5,<2.3 - pytest ; extra == 'test' - pytest-cov ; extra == 'test' - pytest-timeout ; extra == 'test' @@ -5961,7 +5961,7 @@ packages: - cython ; extra == 'test' - meson ; extra == 'test' - ninja ; sys_platform != 'emscripten' and extra == 'test' - - sphinx<=7.3.7,>=5.0.0 ; extra == 'doc' + - sphinx>=5.0.0,<=7.3.7 ; extra == 'doc' - pydata-sphinx-theme>=0.15.2 ; extra == 'doc' - sphinx-design>=0.4.0 ; extra == 'doc' - matplotlib>=3.5 ; extra == 'doc' @@ -6242,7 +6242,7 @@ packages: url: https://files.pythonhosted.org/packages/96/00/2b325970b3060c7cecebab6d295afe763365822b1306a12eeab198f74323/starlette-0.41.3-py3-none-any.whl sha256: 44cedb2b7c77a9de33a8b74b2b90e9f50d11fcf25d8270ea525ad71a25374ff7 requires_dist: - - anyio<5,>=3.4.0 + - anyio>=3.4.0,<5 - typing-extensions>=3.10.0 ; python_full_version < '3.10' - httpx>=0.22.0 ; extra == 'full' - itsdangerous ; extra == 'full' @@ -6602,7 +6602,7 @@ packages: - httptools>=0.6.3 ; extra == 'standard' - python-dotenv>=0.13 ; extra == 'standard' - pyyaml>=5.1 ; extra == 'standard' - - uvloop!=0.15.0,!=0.15.1,>=0.14.0 ; platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32' and extra == 'standard' + - uvloop>=0.14.0,!=0.15.0,!=0.15.1 ; platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32' and extra == 'standard' - watchfiles>=0.13 ; extra == 'standard' - websockets>=10.4 ; extra == 'standard' requires_python: '>=3.8' diff --git a/test/_data.py b/test/_data.py index 562dcb4..1f87211 100644 --- a/test/_data.py +++ b/test/_data.py @@ -14,8 +14,8 @@ from zarr.storage import MemoryStore from deepdiff import DeepDiff -test_data_ipfs_cid = "bafybeiccqqioflsdnpna3kewhskyjcitqdk6n3yrzhnhj5qzpjk5edc2be" -test_data_sha256 = "4921b0e38b09ea480d377a89f1b4074f8e783b5703a651c60ef1e495c877f716" +test_data_ipfs_cid = "bafybeib2s7ls6yscm2uqxby5vbhbfsyxn3ev7soewi3hji4uiki7v6cbiy" +test_data_sha256 = "58c0219f194cd976acee1ebd19ea78b03aada3f96a54302c8fb8515a349d9613" test_dir = Path(__file__).resolve().parent extract_dir = "data" diff --git a/test/test_convert_ome_zarr_version.py b/test/test_convert_ome_zarr_version.py new file mode 100644 index 0000000..a891e45 --- /dev/null +++ b/test/test_convert_ome_zarr_version.py @@ -0,0 +1,40 @@ +from packaging import version +from pathlib import Path + +import pytest + +import zarr.storage +import zarr + +from ngff_zarr import to_ngff_zarr, from_ngff_zarr + + +zarr_version = version.parse(zarr.__version__) + +# Skip tests if zarr version is less than 3.0.0b1 +pytestmark = pytest.mark.skipif( + zarr_version < version.parse("3.0.0b1"), reason="zarr version < 3.0.0b1" +) + + +def test_convert_0_4_to_0_5(): + test_store = Path(__file__).parent / "data" / "input" / "v04" / "6001240.zarr" + multiscales = from_ngff_zarr(test_store, validate=True, version="0.4") + store = zarr.storage.MemoryStore() + version = "0.5" + to_ngff_zarr(store, multiscales, version=version) + from_ngff_zarr(store, validate=True, version=version) + + +def test_convert_0_5_to_0_4(): + test_store = Path(__file__).parent / "data" / "input" / "v04" / "6001240.zarr" + multiscales = from_ngff_zarr(test_store, validate=True, version="0.4") + store = zarr.storage.MemoryStore() + version = "0.5" + to_ngff_zarr(store, multiscales, version=version) + multiscales = from_ngff_zarr(store, validate=True, version=version) + + version = "0.4" + new_store = zarr.storage.MemoryStore() + to_ngff_zarr(new_store, multiscales, version=version) + from_ngff_zarr(new_store, validate=True, version=version) diff --git a/test/test_ngff_validation.py b/test/test_ngff_validation.py index 40e5eb6..4cdb115 100644 --- a/test/test_ngff_validation.py +++ b/test/test_ngff_validation.py @@ -1,4 +1,5 @@ from pathlib import Path +import sys import numpy as np import zarr @@ -41,7 +42,10 @@ def test_y_x_valid_ngff(): def test_validate_0_1(): test_store = Path(__file__).parent / "data" / "input" / "v01" / "6001251.zarr" multiscales = from_ngff_zarr(test_store, validate=True, version="0.1") - print(multiscales) + if sys.byteorder == "little": + assert multiscales.images[0].data.dtype.byteorder == "<" + else: + assert multiscales.images[0].data.dtype.byteorder == ">" def test_validate_0_1_no_version():