From 4d8447170cc9046ecd44005c886745c3a1211b52 Mon Sep 17 00:00:00 2001 From: Fabian Zills <46721498+PythonFZ@users.noreply.github.com> Date: Tue, 21 Mar 2023 14:59:02 +0100 Subject: [PATCH 1/5] update znflow (#540) --- poetry.lock | 131 +++++++++++++++++---------------- pyproject.toml | 2 +- tests/integration/test_misc.py | 25 +++++++ 3 files changed, 94 insertions(+), 64 deletions(-) create mode 100644 tests/integration/test_misc.py diff --git a/poetry.lock b/poetry.lock index 860dfc96..3bfa1d4a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1318,14 +1318,14 @@ pgp = ["gpg"] [[package]] name = "dvc" -version = "2.50.0" +version = "2.51.0" description = "Git for data scientists - manage your code and data together" category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "dvc-2.50.0-py3-none-any.whl", hash = "sha256:9d8b7940a22f7f372104dc81ccd050ae580280538024d4f86ed933f12434a2dc"}, - {file = "dvc-2.50.0.tar.gz", hash = "sha256:4c4c3009e21216e2155a45e006b068c7e3fc9cf9b5440546eafa9bf26fff7878"}, + {file = "dvc-2.51.0-py3-none-any.whl", hash = "sha256:d71a4b1129cbd23e2d00e43ccf0cdf39f60840054d00c7bdc560e2b57c627aab"}, + {file = "dvc-2.51.0.tar.gz", hash = "sha256:4eaacf46d67c5c07ac6866e090da171d13e451cabe97977cdd95b05b7bdf05b6"}, ] [package.dependencies] @@ -1333,9 +1333,9 @@ colorama = ">=0.3.9" configobj = ">=5.0.6" distro = ">=1.3" dpath = ">=2.1.0,<3" -dvc-data = ">=0.44.0,<0.45" +dvc-data = ">=0.44.1,<0.45" dvc-http = "*" -dvc-render = ">=0.1.2" +dvc-render = ">=0.3.1,<0.4.0" dvc-studio-client = ">=0.5.0,<1" dvc-task = ">=0.2.0,<1" flatten-dict = ">=0.4.1,<1" @@ -1974,14 +1974,14 @@ files = [ [[package]] name = "ipykernel" -version = "6.21.3" +version = "6.22.0" description = "IPython Kernel for Jupyter" category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "ipykernel-6.21.3-py3-none-any.whl", hash = "sha256:24ebd9715e317c185e37156ab3a87382410185230dde7aeffce389d6c7d4428a"}, - {file = "ipykernel-6.21.3.tar.gz", hash = "sha256:c8ff581905d70e7299bc1473a2f7c113bec1744fb3746d58e5b4b93bd8ee7001"}, + {file = "ipykernel-6.22.0-py3-none-any.whl", hash = "sha256:1ae6047c1277508933078163721bbb479c3e7292778a04b4bacf0874550977d6"}, + {file = "ipykernel-6.22.0.tar.gz", hash = "sha256:302558b81f1bc22dc259fb2a0c5c7cf2f4c0bdb21b50484348f7bafe7fb71421"}, ] [package.dependencies] @@ -2209,14 +2209,14 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- [[package]] name = "jupyter-client" -version = "8.0.3" +version = "8.1.0" description = "Jupyter protocol implementation and client libraries" category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "jupyter_client-8.0.3-py3-none-any.whl", hash = "sha256:be48ac6bd659cbbddb7a674cf06b3b8afbf53f228253cf58bde604c03bd487b0"}, - {file = "jupyter_client-8.0.3.tar.gz", hash = "sha256:ed65498bea6d876ef9d8da3e0db3dd33c5d129f5b2645f56ae03993782966bd0"}, + {file = "jupyter_client-8.1.0-py3-none-any.whl", hash = "sha256:d5b8e739d7816944be50f81121a109788a3d92732ecf1ad1e4dadebc948818fe"}, + {file = "jupyter_client-8.1.0.tar.gz", hash = "sha256:3fbab64100a0dcac7701b1e0f1a4412f1ccb45546ff2ad9bc4fcbe4e19804811"}, ] [package.dependencies] @@ -2964,14 +2964,14 @@ webpdf = ["pyppeteer (>=1,<1.1)"] [[package]] name = "nbformat" -version = "5.7.3" +version = "5.8.0" description = "The Jupyter Notebook format" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "nbformat-5.7.3-py3-none-any.whl", hash = "sha256:22a98a6516ca216002b0a34591af5bcb8072ca6c63910baffc901cfa07fefbf0"}, - {file = "nbformat-5.7.3.tar.gz", hash = "sha256:4b021fca24d3a747bf4e626694033d792d594705829e5e35b14ee3369f9f6477"}, + {file = "nbformat-5.8.0-py3-none-any.whl", hash = "sha256:d910082bd3e0bffcf07eabf3683ed7dda0727a326c446eeb2922abe102e65162"}, + {file = "nbformat-5.8.0.tar.gz", hash = "sha256:46dac64c781f1c34dfd8acba16547024110348f9fc7eab0f31981c2a3dc48d1f"}, ] [package.dependencies] @@ -3195,56 +3195,61 @@ PyYAML = ">=5.1.0" [[package]] name = "orjson" -version = "3.8.7" +version = "3.8.8" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "orjson-3.8.7-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:f98c82850b7b4b7e27785ca43706fa86c893cdb88d54576bbb9b0d9c1070e421"}, - {file = "orjson-3.8.7-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:1dee503c6c1a0659c5b46f5f39d9ca9d3657b11ca8bb4af8506086df416887d9"}, - {file = "orjson-3.8.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc4fa83831f42ce5c938f8cefc2e175fa1df6f661fdeaba3badf26d2b8cfcf73"}, - {file = "orjson-3.8.7-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9e432c6c9c8b97ad825276d5795286f7cc9689f377a97e3b7ecf14918413303f"}, - {file = "orjson-3.8.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee519964a5a0efb9633f38b1129fd242807c5c57162844efeeaab1c8de080051"}, - {file = "orjson-3.8.7-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:109b539ce5bf60a121454d008fa67c3b67e5a3249e47d277012645922cf74bd0"}, - {file = "orjson-3.8.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ad4d441fbde4133af6fee37f67dbf23181b9c537ecc317346ec8c3b4c8ec7705"}, - {file = "orjson-3.8.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:89dc786419e1ce2588345f58dd6a434e6728bce66b94989644234bcdbe39b603"}, - {file = "orjson-3.8.7-cp310-none-win_amd64.whl", hash = "sha256:697abde7350fb8076d44bcb6b4ab3ce415ae2b5a9bb91efc460e5ab0d96bb5d3"}, - {file = "orjson-3.8.7-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:1c19f47b35b9966a3abadf341b18ee4a860431bf2b00fd8d58906d51cf78aa70"}, - {file = "orjson-3.8.7-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:3ffaabb380cd0ee187b4fc362516df6bf739808130b1339445c7d8878fca36e7"}, - {file = "orjson-3.8.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d88837002c5a8af970745b8e0ca1b0fdb06aafbe7f1279e110d338ea19f3d23"}, - {file = "orjson-3.8.7-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff60187d1b7e0bfab376b6002b08c560b7de06c87cf3a8ac639ecf58f84c5f3b"}, - {file = "orjson-3.8.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0110970aed35dec293f30ed1e09f8604afd5d15c5ef83de7f6c427619b3ba47b"}, - {file = "orjson-3.8.7-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:51b275475d4e36118b65ad56f9764056a09d985c5d72e64579bf8816f1356a5e"}, - {file = "orjson-3.8.7-cp311-none-win_amd64.whl", hash = "sha256:63144d27735f3b60f079f247ac9a289d80dfe49a7f03880dfa0c0ba64d6491d5"}, - {file = "orjson-3.8.7-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:a16273d77db746bb1789a2bbfded81148a60743fd6f9d5185e02d92e3732fa18"}, - {file = "orjson-3.8.7-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:5bb32259ea22cc9dd47a6fdc4b8f9f1e2f798fcf56c7c1122a7df0f4c5d33bf3"}, - {file = "orjson-3.8.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad02e9102d4ba67db30a136e631e32aeebd1dce26c9f5942a457b02df131c5d0"}, - {file = "orjson-3.8.7-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dbcfcec2b7ac52deb7be3685b551addc28ee8fa454ef41f8b714df6ba0e32a27"}, - {file = "orjson-3.8.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1a0e5504a5fc86083cc210c6946e8d61e13fe9f1d7a7bf81b42f7050a49d4fb"}, - {file = "orjson-3.8.7-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:7bd4fd37adb03b1f2a1012d43c9f95973a02164e131dfe3ff804d7e180af5653"}, - {file = "orjson-3.8.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:188ed9f9a781333ad802af54c55d5a48991e292239aef41bd663b6e314377eb8"}, - {file = "orjson-3.8.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:cc52f58c688cb10afd810280e450f56fbcb27f52c053463e625c8335c95db0dc"}, - {file = "orjson-3.8.7-cp37-none-win_amd64.whl", hash = "sha256:403c8c84ac8a02c40613b0493b74d5256379e65196d39399edbf2ed3169cbeb5"}, - {file = "orjson-3.8.7-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:7d6ac5f8a2a17095cd927c4d52abbb38af45918e0d3abd60fb50cfd49d71ae24"}, - {file = "orjson-3.8.7-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:0295a7bfd713fa89231fd0822c995c31fc2343c59a1d13aa1b8b6651335654f5"}, - {file = "orjson-3.8.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feb32aaaa34cf2f891eb793ad320d4bb6731328496ae59b6c9eb1b620c42b529"}, - {file = "orjson-3.8.7-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7a3ab1a473894e609b6f1d763838c6689ba2b97620c256a32c4d9f10595ac179"}, - {file = "orjson-3.8.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e8c430d82b532c5ab95634e034bbf6ca7432ffe175a3e63eadd493e00b3a555"}, - {file = "orjson-3.8.7-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:366cc75f7e09106f9dac95a675aef413367b284f25507d21e55bd7f45f445e80"}, - {file = "orjson-3.8.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:84d154d07e8b17d97e990d5d710b719a031738eb1687d8a05b9089f0564ff3e0"}, - {file = "orjson-3.8.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06180014afcfdc167ca984b312218aa62ce20093965c437c5f9166764cb65ef7"}, - {file = "orjson-3.8.7-cp38-none-win_amd64.whl", hash = "sha256:41244431ba13f2e6ef22b52c5cf0202d17954489f4a3c0505bd28d0e805c3546"}, - {file = "orjson-3.8.7-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:b20f29fa8371b8023f1791df035a2c3ccbd98baa429ac3114fc104768f7db6f8"}, - {file = "orjson-3.8.7-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:226bfc1da2f21ee74918cee2873ea9a0fec1a8830e533cb287d192d593e99d02"}, - {file = "orjson-3.8.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e75c11023ac29e29fd3e75038d0e8dd93f9ea24d7b9a5e871967a8921a88df24"}, - {file = "orjson-3.8.7-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:78604d3acfd7cd502f6381eea0c42281fe2b74755b334074ab3ebc0224100be1"}, - {file = "orjson-3.8.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7129a6847f0494aa1427167486ef6aea2e835ba05f6c627df522692ee228f65"}, - {file = "orjson-3.8.7-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:1a1a8f4980059f48483782c608145b0f74538c266e01c183d9bcd9f8b71dbada"}, - {file = "orjson-3.8.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d60304172a33705ce4bd25a6261ab84bed2dab0b3d3b79672ea16c7648af4832"}, - {file = "orjson-3.8.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4f733062d84389c32c0492e5a4929056fac217034a94523debe0430bcc602cda"}, - {file = "orjson-3.8.7-cp39-none-win_amd64.whl", hash = "sha256:010e2970ec9e826c332819e0da4b14b29b19641da0f1a6af4cec91629ef9b988"}, - {file = "orjson-3.8.7.tar.gz", hash = "sha256:8460c8810652dba59c38c80d27c325b5092d189308d8d4f3e688dbd8d4f3b2dc"}, + {file = "orjson-3.8.8-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:18fcdea75d8b571dc9b185652b81397b62878ae7934fd62e6a0103a5b8448e34"}, + {file = "orjson-3.8.8-cp310-cp310-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:306618884929b596e2e083f82b5617da812df25b0c467542371f1d51f0c5a6f5"}, + {file = "orjson-3.8.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edc65ddb6ae6f8fbb2bbf78ac98f75b729c9eeb0776d5508dd76d3a948dda1dd"}, + {file = "orjson-3.8.8-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e6a6d55e01bce74516dff15302627a13b1f4edcb1c3942dd660978dee423ccf2"}, + {file = "orjson-3.8.8-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:28075c4b502d792fb6703e983d456b2a30d5d6f332d26092eb312dc782e64c64"}, + {file = "orjson-3.8.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eda4c37e48ff549763183a1549c10eec6ea40439520b17d09359cd74a425069"}, + {file = "orjson-3.8.8-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a3eac485a15493164867729f44e1e1247b3094ff19d37708e8cdc9c88a93c623"}, + {file = "orjson-3.8.8-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:88bf40e5468444c04374d1b8f1877cebbaef6bb7406cb6b4a34a570c5cbb87bc"}, + {file = "orjson-3.8.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:747bd4e09d8aa61e1ff677a7dd1cffd28a5d13c22f3769123c58ec988bf1b83d"}, + {file = "orjson-3.8.8-cp310-none-win_amd64.whl", hash = "sha256:dd7d86c5f5f820ac9d4783477e86eb984b63bdb32359935609eb33cf65049c54"}, + {file = "orjson-3.8.8-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:52293a6097750c2d434737966fe6e2a1ed489ac70cc8e584f5944af83de0b787"}, + {file = "orjson-3.8.8-cp311-cp311-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:9322450f392dceb49810d2f820b1932af22d66f67f1d45c31f160067dd06359f"}, + {file = "orjson-3.8.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68d59e3ae84a9b6f14b45a89f7fde4a08a87ea5eb76bfc854b354640de8156f5"}, + {file = "orjson-3.8.8-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:022347dad2253081eaa25366834bb8b06a5aceb0e83b39c6b0aa865759e49d69"}, + {file = "orjson-3.8.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddfcc54793e266056fe1c257d0804c336bca1c5c1ee7979d674e1fc19cfb0a6a"}, + {file = "orjson-3.8.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:449d8ed1e0e6b24e9df5a06b59fd66ea7f7293e141257069601ae8ff9fad705c"}, + {file = "orjson-3.8.8-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0204bc414bc6f7a595211569840b422d96649fd8686efa1fbbcb12eed5dd9521"}, + {file = "orjson-3.8.8-cp311-none-win_amd64.whl", hash = "sha256:e991a5c2c5f2f299c77e1d07ef2812ff5b68e1d97a2aab01aca29cf756473aa3"}, + {file = "orjson-3.8.8-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:2006d9c046bbf335c951f61e016a27bd4f17323dd116f601e4a8a11739cd0a62"}, + {file = "orjson-3.8.8-cp37-cp37m-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:4553d85bad4cbd634a40b7b5d36daaa197a6025f9ce3e2165b371e528759093d"}, + {file = "orjson-3.8.8-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57ee45d2cc6c11c50afb5a0c09d7cd559aea76c77250dbe996be6a03464d4a50"}, + {file = "orjson-3.8.8-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:02f5b5db1e424706eb9f70f1c25699ff4cef16fadfc64af5b70f8628eafe4771"}, + {file = "orjson-3.8.8-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d7c9f3b1598a1ccd806ef02257a76a00c7ede09662ddb54eec2b4bd92874254"}, + {file = "orjson-3.8.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b90d171932b6a9d50e79fa2762cb303e3556bbf25c08bb316fe346ec58af9c19"}, + {file = "orjson-3.8.8-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:28dfe774c345130f1117c4d023644ec52d9d50e3eaadb9bd1c668d91dc109bb5"}, + {file = "orjson-3.8.8-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8f84116fcc3714e7ba3cbeb1b11ac5e4549e7d2726c50142f8299fff9dea7d53"}, + {file = "orjson-3.8.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:f989f8580db86166aaaa938ccd1597ba1817e3f5df14c047baafe783e3d24173"}, + {file = "orjson-3.8.8-cp37-none-win_amd64.whl", hash = "sha256:66045850f286090800a18662d81d44f88c3fcb60ea3a9947d5caeab5d1efc92e"}, + {file = "orjson-3.8.8-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:2c2c5f3d3bbd61dba646e2b9c54a0dd7941b03fba49726bd31c1c23fedf0b9aa"}, + {file = "orjson-3.8.8-cp38-cp38-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:9cb36d4a14f3a911369219d5abc19b907bc41ed2730f7bfe0847b0fd3e834c87"}, + {file = "orjson-3.8.8-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:343124f84da0a33c83ee106a98b3e3c42767c88323d4a2809683cbe83816e8be"}, + {file = "orjson-3.8.8-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24ad122d8dd057acf2a9965a2ffc1bc12fb310ae1cfe2912db930cbb9ef7eaba"}, + {file = "orjson-3.8.8-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2f28a92a9bcb4e8635524b20db1b539bda8613872f306b36cdfd9d3577d03ac"}, + {file = "orjson-3.8.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81d3c5b253eebfc4a61cea1f255a576cb2b889afa99f4510f30ec13201d4f457"}, + {file = "orjson-3.8.8-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:317164f7d4c0540a6eb8b0a0faeec84ef011d359da05188423db762b65f84e1d"}, + {file = "orjson-3.8.8-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5e7e39357371d4ae5649f33c01886508a4c8e5fa5c7344554af041dc0f004c01"}, + {file = "orjson-3.8.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:60fefd4bbd796b4296f478e705fe2c2c7defd28da98d3017743eb87c3238a380"}, + {file = "orjson-3.8.8-cp38-none-win_amd64.whl", hash = "sha256:0dc4a52f1087baeec6b58248fd6b01f17c124fb99f6f770596851ea434a7be0b"}, + {file = "orjson-3.8.8-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:a6bcb449537a99f55c5f05187bac00b4549a795e89c10dcca0d7629548852357"}, + {file = "orjson-3.8.8-cp39-cp39-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:9c98dc791aa44268ba7f6e21124cf885c813b155316c6bf257560571d243fe15"}, + {file = "orjson-3.8.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b2abf93b727a6af7c5ec8816168cbdff39c716af18ced425dd50ae46d69765c"}, + {file = "orjson-3.8.8-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:23447d38375a19d57975d4e32d9ce9f533803c197fd4292e10d3234c052037a8"}, + {file = "orjson-3.8.8-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c2e19d2b46cc93c7218bf8180807bf922ff61dc9883458a06edc66d22970fff"}, + {file = "orjson-3.8.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e480d74d7bf415e6548a364669404119a85dbe0e3c6cd5f7cb4c7003eac20164"}, + {file = "orjson-3.8.8-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:8e0bff5656b99dd975cae2e5230b39e5909d06c0692fd1f6f06dc46f1fe705d0"}, + {file = "orjson-3.8.8-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:56bb6eb7a254eec3b15feba9b20f4172ccbe6ea50a54cf66cbc8e1e4a19585c2"}, + {file = "orjson-3.8.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1af1cfad5d90b68e15fd625c889c4f9f91d7a88f49512cdb89f01c3881e0c9d9"}, + {file = "orjson-3.8.8-cp39-none-win_amd64.whl", hash = "sha256:d5514dfe200356a1d5a6039e00dca78d87d063f3da1eb6a371253e5a8b7ab5b0"}, + {file = "orjson-3.8.8.tar.gz", hash = "sha256:c096d7a523bae6ffb9c4a228ba4691d66113f0f2231579dc945523fbef09c6da"}, ] [[package]] @@ -5214,14 +5219,14 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [[package]] name = "znflow" -version = "0.1.4" +version = "0.1.5" description = "" category = "main" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "znflow-0.1.4-py3-none-any.whl", hash = "sha256:cd90bf5c03161eed5790b6f8efc967d76322a04cf9551be24444f700cf846ebf"}, - {file = "znflow-0.1.4.tar.gz", hash = "sha256:6157997dfcd138837616ee9acb88b231a51bb7515a63bdfc2a1728d6afbe7c5e"}, + {file = "znflow-0.1.5-py3-none-any.whl", hash = "sha256:17bdc7e28f7b3e1802b00aa825248de251b5e2b5db18c9e41c111c8e859efd3f"}, + {file = "znflow-0.1.5.tar.gz", hash = "sha256:dbccdaa0327b56e04d3a51a7159230e6f8ed2f8a2a00b8b034593b046dc8cb9b"}, ] [package.dependencies] @@ -5258,4 +5263,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = ">=3.8,<4.0.0" -content-hash = "16fdfb76cf4c68f77eb4dd14be331687d468a979f768bde118314121d675f8e6" +content-hash = "f79d7f0b0ee47e6ca375581f4af89be73f09439beacef945c0ade0c8597c5b02" diff --git a/pyproject.toml b/pyproject.toml index dbb65cf5..f9889bde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ typer = "^0.7.0" dot4dict = "^0.1.1" zninit = "^0.1.9" -znflow = "^0.1.4" +znflow = "^0.1.5" znjson = "^0.2.2" diff --git a/tests/integration/test_misc.py b/tests/integration/test_misc.py new file mode 100644 index 00000000..40ba2b6c --- /dev/null +++ b/tests/integration/test_misc.py @@ -0,0 +1,25 @@ +import zntrack +import znflow + + +class NodeWithProperty(zntrack.Node): + params = zntrack.zn.params(None) + + @property + def calc(self): + """This should not change the params if not called.""" + self.params = 42 + return "calc" + + def run(self): + pass + + +def test_NodeWithProperty(proj_path): + with zntrack.Project() as proj: + node = NodeWithProperty() + + proj.run() + + node.load() + assert node.params is None From 6226c7365ea20ecfc2afd473a04df0726e9d61b8 Mon Sep 17 00:00:00 2001 From: Fabian Zills <46721498+PythonFZ@users.noreply.github.com> Date: Tue, 21 Mar 2023 16:54:23 +0100 Subject: [PATCH 2/5] fix zn.nodes(None) (#541) --- tests/integration/test_none_values.py | 10 ++++++++-- zntrack/fields/zn/__init__.py | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_none_values.py b/tests/integration/test_none_values.py index 14f2b6da..f34b36d9 100644 --- a/tests/integration/test_none_values.py +++ b/tests/integration/test_none_values.py @@ -66,11 +66,17 @@ def test_from_dvc_deps(proj_path, eager): class EmptyNodesNode(zntrack.Node): nodes = zntrack.zn.nodes(None) + outs = zntrack.zn.outs() def run(self): - pass + self.outs = 42 -def test_EmptyNode(proj_path): +@pytest.mark.parametrize("eager", [True, False]) +def test_EmptyNode(proj_path, eager): with zntrack.Project() as project: node = EmptyNodesNode() + project.run(eager=eager) + if not eager: + node.load() + assert node.outs == 42 diff --git a/zntrack/fields/zn/__init__.py b/zntrack/fields/zn/__init__.py index dbfd760a..857e2258 100644 --- a/zntrack/fields/zn/__init__.py +++ b/zntrack/fields/zn/__init__.py @@ -369,6 +369,8 @@ def __set__(self, instance, value): def get_node_names(self, instance) -> list: """Get the name of the other Node.""" value = instance.__dict__[self.name] + if value is None: # the zn.nodes(None) case + return [] if isinstance(value, (list, tuple)): return [f"{instance.name}_{self.name}_{idx}" for idx in range(len(value))] return [f"{instance.name}_{self.name}"] From 3456e5f3f4bbe3f19ee3746e9f70fcee0628b540 Mon Sep 17 00:00:00 2001 From: Fabian Zills <46721498+PythonFZ@users.noreply.github.com> Date: Tue, 21 Mar 2023 22:38:30 +0100 Subject: [PATCH 3/5] Fix issue with `zn.nodes` (#542) * add fix * bugfix * add parent method * do not return default value * use contextlib * return None, if AttributeError * fix None type and older python version --- tests/integration/test_none_values.py | 8 ++++- zntrack/fields/field.py | 21 ++++++++++++ zntrack/fields/zn/__init__.py | 49 +++++++++++++-------------- 3 files changed, 51 insertions(+), 27 deletions(-) diff --git a/tests/integration/test_none_values.py b/tests/integration/test_none_values.py index f34b36d9..9df10ca2 100644 --- a/tests/integration/test_none_values.py +++ b/tests/integration/test_none_values.py @@ -65,11 +65,17 @@ def test_from_dvc_deps(proj_path, eager): class EmptyNodesNode(zntrack.Node): + # we use dvc.outs to generate zntrack.json + file = zntrack.dvc.outs(zntrack.nwd / "file.txt") nodes = zntrack.zn.nodes(None) outs = zntrack.zn.outs() def run(self): - self.outs = 42 + if self.nodes is None: + self.outs = 42 + else: + self.outs = self.nodes.value + self.file.write_text("Hello World") @pytest.mark.parametrize("eager", [True, False]) diff --git a/zntrack/fields/field.py b/zntrack/fields/field.py index fd67356d..b4f50069 100644 --- a/zntrack/fields/field.py +++ b/zntrack/fields/field.py @@ -1,5 +1,6 @@ """The base class for all fields.""" import abc +import contextlib import enum import json import logging @@ -149,9 +150,29 @@ def _write_value_to_config(self, value, instance: "Node", encoder=None): json.dump(zntrack_dict, f, indent=4, cls=encoder) +class DataIsLazyError(Exception): + """Exception to raise when a field is accessed that contains lazy data.""" + + class LazyField(Field): """Base class for fields that are loaded lazily.""" + def get_value_except_lazy(self, instance): + """Get the value of the field. + + If the value is lazy, raise an Error. + + Raises + ------ + DataIsLazyError + If the value is lazy. + """ + with contextlib.suppress(KeyError): + if instance.__dict__[self.name] is LazyOption: + raise DataIsLazyError() + + return getattr(instance, self.name, None) + def __get__(self, instance, owner=None): """Load the field from disk if it is not already loaded.""" if instance is None: diff --git a/zntrack/fields/zn/__init__.py b/zntrack/fields/zn/__init__.py index 857e2258..48338bfd 100644 --- a/zntrack/fields/zn/__init__.py +++ b/zntrack/fields/zn/__init__.py @@ -12,8 +12,8 @@ import znjson from znflow import handler -from zntrack.fields.field import Field, FieldGroup, LazyField -from zntrack.utils import LazyOption, module_handler, update_key_val +from zntrack.fields.field import DataIsLazyError, Field, FieldGroup, LazyField +from zntrack.utils import module_handler, update_key_val if typing.TYPE_CHECKING: from zntrack import Node @@ -169,11 +169,8 @@ def save(self, instance: "Node"): The node instance. """ try: - value = getattr(instance, self.name) - except AttributeError: - return - - if value is LazyOption: + value = self.get_value_except_lazy(instance) + except DataIsLazyError: return instance.nwd.mkdir(exist_ok=True, parents=True) @@ -218,11 +215,10 @@ def get_files(self, instance) -> list: def save(self, instance: "Node"): """Save the field to disk.""" try: - value: pd.DataFrame = getattr(instance, self.name) - except AttributeError: + value = self.get_value_except_lazy(instance) + except DataIsLazyError: return - - if value is LazyOption: + if value is None: return instance.nwd.mkdir(exist_ok=True, parents=True) @@ -294,11 +290,8 @@ def get_files(self, instance) -> list: def save(self, instance: "Node"): """Save the field to disk.""" try: - value = instance.__dict__[self.name] - except KeyError: - return - - if value is LazyOption: + value = self.get_value_except_lazy(instance) + except DataIsLazyError: return self._write_value_to_config( @@ -368,7 +361,11 @@ def __set__(self, instance, value): def get_node_names(self, instance) -> list: """Get the name of the other Node.""" - value = instance.__dict__[self.name] + try: + value = self.get_value_except_lazy(instance) + except DataIsLazyError: + return [] + if value is None: # the zn.nodes(None) case return [] if isinstance(value, (list, tuple)): @@ -378,21 +375,21 @@ def get_node_names(self, instance) -> list: def save(self, instance: "Node"): """Save the Node parameters to disk.""" try: - value = getattr(instance, self.name) - except AttributeError: - return - if value in [LazyOption, None]: + value = self.get_value_except_lazy(instance) + except DataIsLazyError: return - if not isinstance(value, (list, tuple)): - value = [value] - for node, name in zip(value, self.get_node_names(instance)): - _SaveNodes()(node, name=name) + if value is not None: + if not isinstance(value, (list, tuple)): + value = [value] + + for node, name in zip(value, self.get_node_names(instance)): + _SaveNodes()(node, name=name) super().save(instance) def get_optional_dvc_cmd(self, instance: "Node") -> typing.List[list]: """Get the dvc command for this field.""" - nodes = instance.__dict__[self.name] + nodes = getattr(instance, self.name) if nodes is None: return [] From fb922c6afe75f5f5285a955d734c75b93bfc5c63 Mon Sep 17 00:00:00 2001 From: Fabian Zills <46721498+PythonFZ@users.noreply.github.com> Date: Wed, 22 Mar 2023 18:44:32 +0100 Subject: [PATCH 4/5] add remove existing graph arg to zntrack.Project (#544) * add exception and remove_existing_graph * change check --- tests/integration/test_project.py | 16 ++++++++++++++++ zntrack/__init__.py | 3 ++- zntrack/core/node.py | 12 ++++++++---- zntrack/exceptions/__init__.py | 19 +++++++++++++++++++ zntrack/project/zntrack_project.py | 12 +++++++++++- 5 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 zntrack/exceptions/__init__.py diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index ae7f4e23..395dc839 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -71,3 +71,19 @@ def test_WriteIO_no_name(tmp_path_2, assert_before_exp): assert exp2["WriteIO"].inputs == "Lorem Ipsum" assert exp2["WriteIO"].outputs == "Lorem Ipsum" + + +def test_project_remove_graph(proj_path): + with zntrack.Project() as project: + node = WriteIO(inputs="Hello World") + project.run() + node.load() + assert node.outputs == "Hello World" + + with zntrack.Project(remove_existing_graph=True) as project: + node2 = WriteIO(inputs="Lorem Ipsum", name="node2") + project.run() + node2.load() + assert node2.outputs == "Lorem Ipsum" + with pytest.raises(zntrack.exceptions.NodeNotAvailableError): + node.load() diff --git a/zntrack/__init__.py b/zntrack/__init__.py index f1e67eee..cff88f76 100644 --- a/zntrack/__init__.py +++ b/zntrack/__init__.py @@ -4,7 +4,7 @@ """ import importlib.metadata -from zntrack import tools +from zntrack import exceptions, tools from zntrack.core.node import Node from zntrack.core.nodify import NodeConfig, nodify from zntrack.fields import Field, FieldGroup, LazyField, dvc, meta, zn @@ -28,4 +28,5 @@ "nodify", "NodeConfig", "tools", + "exceptions", ] diff --git a/zntrack/core/node.py b/zntrack/core/node.py index a78fd152..3f0c2366 100644 --- a/zntrack/core/node.py +++ b/zntrack/core/node.py @@ -14,6 +14,7 @@ import zninit import znjson +from zntrack import exceptions from zntrack.notebooks.jupyter import jupyter_class_to_file from zntrack.utils import NodeStatusResults, deprecated, module_handler, run_dvc_cmd from zntrack.utils.config import config @@ -162,10 +163,13 @@ def load(self, lazy: bool = None) -> None: kwargs = {} if lazy is None else {"lazy": lazy} self.state.loaded = True # we assume loading will be successful. - with config.updated_config(**kwargs): - # TODO: it would be much nicer not to use a global config object here. - for attr in zninit.get_descriptors(Field, self=self): - attr.load(self) + try: + with config.updated_config(**kwargs): + # TODO: it would be much nicer not to use a global config object here. + for attr in zninit.get_descriptors(Field, self=self): + attr.load(self) + except KeyError as err: + raise exceptions.NodeNotAvailableError(self) from err # TODO: documentation about _post_init and _post_load_ and when they are called self._post_load_() diff --git a/zntrack/exceptions/__init__.py b/zntrack/exceptions/__init__.py new file mode 100644 index 00000000..3408c03f --- /dev/null +++ b/zntrack/exceptions/__init__.py @@ -0,0 +1,19 @@ +"""All ZnTrack exceptions.""" + + +class NodeNotAvailableError(Exception): + """Raised when a node is not available.""" + + def __init__(self, arg): + """Initialize the exception. + + Parameters + ---------- + arg : str|Node + Custom Error message or Node that is not available. + """ + if isinstance(arg, str): + super().__init__(arg) + else: + # assume arg is a Node + super().__init__(f"Node {arg.name} is not available.") diff --git a/zntrack/project/zntrack_project.py b/zntrack/project/zntrack_project.py index aa36b9a1..c5b650f0 100644 --- a/zntrack/project/zntrack_project.py +++ b/zntrack/project/zntrack_project.py @@ -96,19 +96,29 @@ def _initalize(): class Project(_ProjectBase): """The ZnTrack Project class.""" - def __init__(self, initialize: bool = True) -> None: + def __init__( + self, initialize: bool = True, remove_existing_graph: bool = False + ) -> None: """Initialize the Project. Attributes ---------- initialize : bool, default = True If True, initialize a git repository and a dvc repository. + remove_existing_graph : bool, default = False + If True, remove 'dvc.yaml', 'zntrack.json' and 'params.yaml' + before writing new nodes. """ # TODO maybe it is not a good idea to base everything on the DiGraph class. # It seems to call some class methods super().__init__() if initialize: _initalize() + if remove_existing_graph: + # we remove the files that typically contain the graph definition + pathlib.Path("zntrack.json").unlink(missing_ok=True) + pathlib.Path("dvc.yaml").unlink(missing_ok=True) + pathlib.Path("params.yaml").unlink(missing_ok=True) def create_branch(self, name: str) -> "Branch": """Create a branch in the project.""" From 44deaa90190a18f94a1790202ba28045b7884385 Mon Sep 17 00:00:00 2001 From: Fabian Zills <46721498+PythonFZ@users.noreply.github.com> Date: Wed, 22 Mar 2023 19:16:57 +0100 Subject: [PATCH 5/5] Update znflow (#548) * add eager test * first try in supporting AddedConnections * convert tuple to list * use znflow from pypi * add test for combined dict * add error message * add comments * use dict comprehension * add 'name' to '_protected_' --- poetry.lock | 16 ++++--- pyproject.toml | 2 +- tests/integration/test_combine_lists.py | 59 +++++++++++++++++++++++++ tests/integration/test_misc.py | 3 +- zntrack/core/node.py | 2 + zntrack/fields/zn/__init__.py | 49 +++++++++++++++++++- 6 files changed, 120 insertions(+), 11 deletions(-) create mode 100644 tests/integration/test_combine_lists.py diff --git a/poetry.lock b/poetry.lock index 3bfa1d4a..28200c9a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -873,21 +873,23 @@ files = [ [[package]] name = "comm" -version = "0.1.2" +version = "0.1.3" description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc." category = "dev" optional = false python-versions = ">=3.6" files = [ - {file = "comm-0.1.2-py3-none-any.whl", hash = "sha256:9f3abf3515112fa7c55a42a6a5ab358735c9dccc8b5910a9d8e3ef5998130666"}, - {file = "comm-0.1.2.tar.gz", hash = "sha256:3e2f5826578e683999b93716285b3b1f344f157bf75fa9ce0a797564e742f062"}, + {file = "comm-0.1.3-py3-none-any.whl", hash = "sha256:16613c6211e20223f215fc6d3b266a247b6e2641bf4e0a3ad34cb1aff2aa3f37"}, + {file = "comm-0.1.3.tar.gz", hash = "sha256:a61efa9daffcfbe66fd643ba966f846a624e4e6d6767eda9cf6e993aadaab93e"}, ] [package.dependencies] traitlets = ">=5.3" [package.extras] +lint = ["black (>=22.6.0)", "mdformat (>0.7)", "mdformat-gfm (>=0.3.5)", "ruff (>=0.0.156)"] test = ["pytest"] +typing = ["mypy (>=0.990)"] [[package]] name = "configobj" @@ -5219,14 +5221,14 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [[package]] name = "znflow" -version = "0.1.5" +version = "0.1.6" description = "" category = "main" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "znflow-0.1.5-py3-none-any.whl", hash = "sha256:17bdc7e28f7b3e1802b00aa825248de251b5e2b5db18c9e41c111c8e859efd3f"}, - {file = "znflow-0.1.5.tar.gz", hash = "sha256:dbccdaa0327b56e04d3a51a7159230e6f8ed2f8a2a00b8b034593b046dc8cb9b"}, + {file = "znflow-0.1.6-py3-none-any.whl", hash = "sha256:5d13feac56b45e0f0e574e1563620ae135b27d083a5c54d4488c3783a4e56830"}, + {file = "znflow-0.1.6.tar.gz", hash = "sha256:2864f4a3f2f9c6a001cfc5ff7410bdac42d493accd1d78a90b2089275a0a00d7"}, ] [package.dependencies] @@ -5263,4 +5265,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = ">=3.8,<4.0.0" -content-hash = "f79d7f0b0ee47e6ca375581f4af89be73f09439beacef945c0ade0c8597c5b02" +content-hash = "4c5437db27f5d39d7e6923c56575f81515179b9b5a49e2526abef015589181db" diff --git a/pyproject.toml b/pyproject.toml index f9889bde..bfd7da38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,8 +18,8 @@ typer = "^0.7.0" dot4dict = "^0.1.1" zninit = "^0.1.9" -znflow = "^0.1.5" znjson = "^0.2.2" +znflow = "^0.1.6" [tool.poetry.urls] diff --git a/tests/integration/test_combine_lists.py b/tests/integration/test_combine_lists.py new file mode 100644 index 00000000..9026c841 --- /dev/null +++ b/tests/integration/test_combine_lists.py @@ -0,0 +1,59 @@ +import pytest + +import zntrack + + +class GenerateList(zntrack.Node): + size = zntrack.zn.params(10) + outs = zntrack.zn.outs() + + def run(self): + self.outs = list(range(self.size)) + + +class AddOneToList(zntrack.Node): + data = zntrack.zn.deps() + outs = zntrack.zn.outs() + + def run(self) -> None: + self.outs = [x + 1 for x in self.data] + + +class AddOneToDict(zntrack.Node): + data = zntrack.zn.deps() + outs = zntrack.zn.outs() + + def run(self) -> None: + self.outs = {k: [x + 1 for x in v] for k, v in self.data.items()} + + +@pytest.mark.parametrize("eager", [True, False]) +def test_combine(proj_path, eager): + with zntrack.Project() as proj: + a = GenerateList(size=1, name="a") + b = GenerateList(size=2, name="b") + c = GenerateList(size=3, name="c") + + added = AddOneToList(data=a.outs + b.outs + c.outs) + + proj.run(eager=eager) + if not eager: + added.load() + + assert added.outs == [1] + [1, 2] + [1, 2, 3] + + +@pytest.mark.parametrize("eager", [True, False]) +def test_combine_dict(proj_path, eager): + with zntrack.Project() as proj: + a = GenerateList(size=1, name="a") + b = GenerateList(size=2, name="b") + c = GenerateList(size=3, name="c") + + added = AddOneToDict(data={x.name: x.outs for x in [a, b, c]}) + + proj.run(eager=eager) + if not eager: + added.load() + + assert added.outs == {"a": [1], "b": [1, 2], "c": [1, 2, 3]} diff --git a/tests/integration/test_misc.py b/tests/integration/test_misc.py index 40ba2b6c..4c320b82 100644 --- a/tests/integration/test_misc.py +++ b/tests/integration/test_misc.py @@ -1,6 +1,7 @@ -import zntrack import znflow +import zntrack + class NodeWithProperty(zntrack.Node): params = zntrack.zn.params(None) diff --git a/zntrack/core/node.py b/zntrack/core/node.py index 3f0c2366..5eacbf5e 100644 --- a/zntrack/core/node.py +++ b/zntrack/core/node.py @@ -88,6 +88,8 @@ class Node(zninit.ZnInit, znflow.Node): name: str = _NameDescriptor(None) _name_ = None + _protected_ = znflow.Node._protected_ + ["name"] + def _post_load_(self) -> None: """Post load hook. diff --git a/zntrack/fields/zn/__init__.py b/zntrack/fields/zn/__init__.py index 48338bfd..4ee26ae8 100644 --- a/zntrack/fields/zn/__init__.py +++ b/zntrack/fields/zn/__init__.py @@ -7,6 +7,7 @@ import pandas as pd import yaml +import znflow import znflow.utils import zninit import znjson @@ -38,6 +39,36 @@ def decode(self, value: str) -> znflow.Connection: return znflow.Connection(**value) +class CombinedConnectionsConverter(znjson.ConverterBase): + """Convert a znflow.Connection object to dict and back.""" + + level = 100 + representation = "znflow.CombinedConnections" + instance = znflow.CombinedConnections + + def encode(self, obj: znflow.CombinedConnections) -> dict: + """Convert the znflow.Connection object to dict.""" + if obj.item is not None: + raise NotImplementedError( + "znflow.CombinedConnections getitem is not supported yet." + ) + return dataclasses.asdict(obj) + + def decode(self, value: str) -> znflow.CombinedConnections: + """Create znflow.Connection object from dict.""" + connections = [] + for item in value["connections"]: + if isinstance(item, dict): + # @nodify functions aren't support as 'zn.deps' + # Nodes directly aren't supported because they aren't lists + connections.append(znflow.Connection(**item)) + else: + # For the case that item is already a znflow.Connection + connections.append(item) + value["connections"] = connections + return znflow.CombinedConnections(**value) + + class SliceConverter(znjson.ConverterBase): """Convert a znflow.Connection object to dict and back.""" @@ -269,9 +300,21 @@ def get_files(self, instance) -> list: files = [] value = getattr(instance, self.name) + # TODO use IterableHandler? + if isinstance(value, dict): + value = list(value.values()) if not isinstance(value, (list, tuple)): value = [value] + if isinstance(value, tuple): + value = list(value) + + others = [] + for node in value: + if isinstance(node, znflow.CombinedConnections): + others.extend(node.connections) + + value.extend(others) for node in value: if node is None: @@ -298,7 +341,7 @@ def save(self, instance: "Node"): value, instance, encoder=znjson.ZnEncoder.from_converters( - [ConnectionConverter], add_default=True + [ConnectionConverter, CombinedConnectionsConverter], add_default=True ), ) @@ -313,7 +356,9 @@ def get_data(self, instance: "Node") -> any: value = json.loads( json.dumps(value), - cls=znjson.ZnDecoder.from_converters(ConnectionConverter, add_default=True), + cls=znjson.ZnDecoder.from_converters( + [ConnectionConverter, CombinedConnectionsConverter], add_default=True + ), ) # Up until here we have connection objects. Now we need