From 5f54a77d1d39dad2fc0841a609164ac1c7d276c0 Mon Sep 17 00:00:00 2001 From: Joe Lynch Date: Fri, 27 Sep 2024 08:57:43 +0000 Subject: [PATCH 1/2] Package for f40 The main change here is that we need to work for both pydantic 1.10.17 and pydantic 2. --- .github/workflows/test.yml | 9 ++-- Dockerfile.fedora | 2 +- astacus/common/cassandra/config.py | 2 +- astacus/common/m3placement.py | 2 +- astacus/common/msgspec_glue.py | 6 +-- astacus/common/rohmustorage.py | 2 +- astacus/common/utils.py | 2 +- .../coordinator/plugins/cassandra/model.py | 2 +- .../coordinator/plugins/zookeeper_config.py | 2 +- astacus/node/cassandra.py | 2 +- astacus/node/config.py | 2 +- pyproject.toml | 47 ++++++++----------- .../plugins/clickhouse/test_config.py | 2 +- 13 files changed, 37 insertions(+), 45 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0a913267..242ec420 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,14 +12,13 @@ jobs: strategy: max-parallel: 4 matrix: - fedora-version: ["38", "39"] + fedora-version: ["39", "40"] include: - - fedora-version: "38" - python-version: "3.11" - fedora-version: "39" - # https://github.com/pydantic/pydantic/issues/9637 - python-version: "3.12.3" + python-version: "3.12.6" + - fedora-version: "40" + python-version: "3.12.6" steps: - uses: actions/checkout@v3 diff --git a/Dockerfile.fedora b/Dockerfile.fedora index 93f2a1b3..dd3b022d 100644 --- a/Dockerfile.fedora +++ b/Dockerfile.fedora @@ -1,6 +1,6 @@ # -*- dockerfile -*- -FROM fedora:38 +FROM fedora:40 MAINTAINER "Markus Stenberg " RUN dnf install -y sudo make diff --git a/astacus/common/cassandra/config.py b/astacus/common/cassandra/config.py index a54778af..b65403b2 100644 --- a/astacus/common/cassandra/config.py +++ b/astacus/common/cassandra/config.py @@ -19,7 +19,7 @@ from astacus.common.utils import AstacusModel from collections.abc import Sequence from pathlib import Path -from pydantic import root_validator +from pydantic.v1 import root_validator import yaml diff --git a/astacus/common/m3placement.py b/astacus/common/m3placement.py index 2e5fdb92..a0c9de4e 100644 --- a/astacus/common/m3placement.py +++ b/astacus/common/m3placement.py @@ -14,7 +14,7 @@ from astacus.common.utils import AstacusModel from astacus.proto import m3_placement_pb2 from collections.abc import Sequence -from pydantic import validator +from pydantic.v1 import validator MAXIMUM_PROTOBUF_STR_LENGTH = 127 diff --git a/astacus/common/msgspec_glue.py b/astacus/common/msgspec_glue.py index c571cdf3..83848c71 100644 --- a/astacus/common/msgspec_glue.py +++ b/astacus/common/msgspec_glue.py @@ -3,9 +3,9 @@ See LICENSE for details """ -from pydantic import PydanticValueError -from pydantic.fields import ModelField -from pydantic.validators import _VALIDATORS +from pydantic.v1 import PydanticValueError +from pydantic.v1.fields import ModelField +from pydantic.v1.validators import _VALIDATORS from starlette.responses import JSONResponse from typing import Any diff --git a/astacus/common/rohmustorage.py b/astacus/common/rohmustorage.py index 44a0b111..598221ee 100644 --- a/astacus/common/rohmustorage.py +++ b/astacus/common/rohmustorage.py @@ -15,7 +15,7 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey from enum import Enum -from pydantic import Field +from pydantic.v1 import Field from rohmu import errors, rohmufile from rohmu.compressor import CompressionStream from rohmu.encryptor import EncryptorStream diff --git a/astacus/common/utils.py b/astacus/common/utils.py index c21c6e66..abecf5b5 100644 --- a/astacus/common/utils.py +++ b/astacus/common/utils.py @@ -17,7 +17,7 @@ from contextlib import contextmanager from multiprocessing.dummy import Pool # fastapi + fork = bad idea from pathlib import Path -from pydantic import BaseModel +from pydantic.v1 import BaseModel from typing import Any, ContextManager, Final, Generic, IO, Literal, overload, TextIO, TypeAlias, TypeVar import asyncio diff --git a/astacus/coordinator/plugins/cassandra/model.py b/astacus/coordinator/plugins/cassandra/model.py index 539ddf68..7adf9ae9 100644 --- a/astacus/coordinator/plugins/cassandra/model.py +++ b/astacus/coordinator/plugins/cassandra/model.py @@ -8,7 +8,7 @@ from astacus.common.cassandra.schema import CassandraSchema from astacus.common.utils import AstacusModel from collections.abc import Sequence -from pydantic import root_validator +from pydantic.v1 import root_validator from uuid import UUID diff --git a/astacus/coordinator/plugins/zookeeper_config.py b/astacus/coordinator/plugins/zookeeper_config.py index 949cf725..6dcc2a72 100644 --- a/astacus/coordinator/plugins/zookeeper_config.py +++ b/astacus/coordinator/plugins/zookeeper_config.py @@ -6,7 +6,7 @@ from astacus.common.utils import AstacusModel, build_netloc from astacus.coordinator.plugins.zookeeper import KazooZooKeeperClient, ZooKeeperClient, ZooKeeperUser from collections.abc import Sequence -from pydantic import SecretStr +from pydantic.v1 import SecretStr class ZooKeeperNode(AstacusModel): diff --git a/astacus/node/cassandra.py b/astacus/node/cassandra.py index 171f57de..eeab1dd9 100644 --- a/astacus/node/cassandra.py +++ b/astacus/node/cassandra.py @@ -14,7 +14,7 @@ from astacus.common.exceptions import TransientException from collections.abc import Callable from pathlib import Path -from pydantic import DirectoryPath +from pydantic.v1 import DirectoryPath import contextlib import logging diff --git a/astacus/node/config.py b/astacus/node/config.py index e80dcd11..f83d79d1 100644 --- a/astacus/node/config.py +++ b/astacus/node/config.py @@ -11,7 +11,7 @@ from collections.abc import Sequence from fastapi import Request from pathlib import Path -from pydantic import DirectoryPath, Field, validator +from pydantic.v1 import DirectoryPath, Field, validator APP_KEY = "node_config" diff --git a/pyproject.toml b/pyproject.toml index 2a2c710c..4550771c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "msgspec", "kazoo", "protobuf < 3.21", - "pydantic < 2", + "pydantic >= 1.10.17", "pyyaml", "rohmu >= 2.7.0", "sentry-sdk", @@ -43,58 +43,51 @@ dependencies = [ cassandra = [ "cassandra-driver == 3.20.2", ] -f38 = [ +f39 = [ "cramjam == 2.8.3", - "cryptography == 37.0.2", - "fastapi == 0.99.0", - # h11 on Fedora 38 is actually 0.13.0, but 0.13.0 is rejeceted by httpcore 0.15.0 - # See: https://github.com/encode/httpcore/blob/0.15.0/setup.py#L56-L57 - # See also: https://github.com/python-hyper/h11/compare/v0.12.0...v0.13.0 if you're looking for meaningful differences. - "h11 == 0.12.0", - "httpcore == 0.15.0", - "httplib2 == 0.20.4", + "cryptography == 41.0.7", + "fastapi == 0.103.0", + "h11 == 0.14.0", + "httpcore == 0.17.3", + "httplib2 == 0.21.0", "httpx == 0.24.1", "msgspec == 0.18.6", "kazoo == 2.8.0", "protobuf == 3.19.6", - # pydantic on Fedora 38 is actually 1.10.2, but 1.10.2 is incompatible with - # mypy >= 1.4.0, this was fixed in pydantic 1.10.9: https://github.com/pydantic/pydantic/pull/5928 - # Further than that, rohmu >= 2.5.0 requires pydantic >= 1.10.17 because of the "v1" namespace broken compatibility. + "pyasyncore == 1.0.2", + # pydantic on Fedora 39 is actually 1.10.14. + # rohmu requires pydantic >= 1.10.17 because of the "v1" namespace broken compatibility. "pydantic == 1.10.17", - "pyyaml == 6.0.0", + "pyyaml == 6.0.1", "requests == 2.28.2", "starlette == 0.27.0", "tabulate == 0.9.0", "uritemplate == 4.1.1", "urllib3 == 1.26.18", - "uvicorn == 0.15.0", + "uvicorn == 0.23.2", "wcmatch == 8.4.1", "zstandard == 0.21.0", ] -f39 = [ +f40 = [ "cramjam == 2.8.3", "cryptography == 41.0.7", - "fastapi == 0.103.0", + "fastapi == 0.111.1", "h11 == 0.14.0", - "httpcore == 0.17.3", + "httpcore == 1.0.2", "httplib2 == 0.21.0", - "httpx == 0.24.1", + "httpx == 0.26.0", "msgspec == 0.18.6", "kazoo == 2.8.0", "protobuf == 3.19.6", "pyasyncore == 1.0.2", - # pydantic on Fedora 39 is actually 1.10.14. - # rohmu requires pydantic >= 1.10.17 because of the "v1" namespace broken compatibility. - "pydantic == 1.10.17", + "pydantic == 2.8.2", "pyyaml == 6.0.1", - "requests == 2.28.2", - "starlette == 0.27.0", + "requests == 2.31.0", "tabulate == 0.9.0", "uritemplate == 4.1.1", - "urllib3 == 1.26.18", "uvicorn == 0.23.2", - "wcmatch == 8.4.1", - "zstandard == 0.21.0", + "wcmatch == 8.5.2", + "zstandard == 0.22.0", ] dev = [ # Needed by pre-commit to lint and test the project diff --git a/tests/unit/coordinator/plugins/clickhouse/test_config.py b/tests/unit/coordinator/plugins/clickhouse/test_config.py index 89d069c3..12783ebf 100644 --- a/tests/unit/coordinator/plugins/clickhouse/test_config.py +++ b/tests/unit/coordinator/plugins/clickhouse/test_config.py @@ -14,7 +14,7 @@ ) from collections.abc import Sequence from kazoo.client import KazooClient -from pydantic import SecretStr +from pydantic.v1 import SecretStr from typing import cast import pytest From b433d8cbd4b2bab212a304170ae84fa633edca33 Mon Sep 17 00:00:00 2001 From: Joe Lynch Date: Fri, 27 Sep 2024 11:17:19 +0000 Subject: [PATCH 2/2] Migrate mypy to pyproject.toml * Bump mypy version * Use pydantic.v1 mypy plugin --- astacus/common/progress.py | 3 +- mypy.ini | 94 -------------------------------------- pyproject.toml | 43 ++++++++++++++++- 3 files changed, 44 insertions(+), 96 deletions(-) delete mode 100644 mypy.ini diff --git a/astacus/common/progress.py b/astacus/common/progress.py index ef95e815..1866e932 100644 --- a/astacus/common/progress.py +++ b/astacus/common/progress.py @@ -59,7 +59,8 @@ def wrap(self, i: Sequence[T]) -> Iterator[T]: self.add_total(len(i)) except TypeError: # Can't compute progress for this iterator - return i + yield from i + return None for item in i: yield item diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index c4aa7244..00000000 --- a/mypy.ini +++ /dev/null @@ -1,94 +0,0 @@ -[mypy] -python_version = 3.11 -plugins = pydantic.mypy -exclude = (?x)( - ^setup.py - | ^vendor/ - | ^venv/ - | ^astacus/proto/ - ) - -show_error_codes = True - -# Dead code is bad -warn_unreachable = True - -# Re-export is bit bad - __all__ should be used if we really want re-export -no_implicit_reexport = True - -# casting is bad, unnecessary casting worse -warn_redundant_casts = True - -# unused type disables are confusing -warn_unused_ignores = True - -# Replacing typing with non-typing is bad -# (This is quite painful to do though, TBD later) -#disallow_untyped_decorators = True - -# Any subclassing is probably an error -disallow_subclassing_any = true - -# This codebase should be typed -#disallow_incomplete_defs = true -#disallow_untyped_defs = true - -# configure pydantic.mypy plugin -[pydantic-mypy] - -# even if Config.extra != forbid, prevent extra args -init_forbid_extra = True - -# validate types on __init__, do not go for type coercion -init_typed = True - -# We don't want use of indirect population without explicit alias -warn_required_dynamic_aliases=True - -# We want types for all fields -# (but some hacks like validators do not work with this well, sadly) -#warn_untyped_fields=True - - -# Ignores start here (toml format would be nicer but guess this works) - -[mypy-astacus.proto.*] -ignore_errors = True - -[mypy-cassandra.*] -ignore_missing_imports = True - -[mypy-google.protobuf.*] -ignore_errors = True - -[mypy-graphlib.*] -ignore_missing_imports = True - -# types-kazoo does not unfortunately fix this -[mypy-kazoo.*] -ignore_missing_imports = True - -# httpcore dependency -[mypy-h11.*] -ignore_missing_imports = True - -[mypy-systemd.*] -ignore_missing_imports = True - -# uvicorn has types starting version 0.19, but we are using 0.15 -[mypy-uvicorn.*] -ignore_missing_imports = True - -# TBD: When someone has time, implement these for whole codebase - -[mypy-astacus.common.cassandra.*] -disallow_incomplete_defs = true -disallow_untyped_defs = true - -[mypy-astacus.coordinator.plugins.cassandra.*] -disallow_incomplete_defs = true -disallow_untyped_defs = true - -[mypy-astacus.node.cassandra] -disallow_incomplete_defs = true -disallow_untyped_defs = true diff --git a/pyproject.toml b/pyproject.toml index 4550771c..00c9ab38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ classifiers=[ "Intended Audience :: System Administrators", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Database :: Database Engines/Servers", "Topic :: Software Development :: Libraries", ] @@ -100,7 +101,7 @@ dev = [ "pytest-timeout==2.1.0", "pytest-watch==4.2.0", "pytest==7.2.2", - "mypy==1.9.0", + "mypy==1.11.2", # Types for things that don't seem to have them "types-botocore>=1.0.2", "types-PyYAML>=6.0.12.2", @@ -147,3 +148,43 @@ source = [ "astacus", "tests" ] + +[tool.mypy] +python_version = "3.12" +plugins = [ + "pydantic.v1.mypy", + "pydantic.mypy", +] +exclude = [ + "setup.py", + "vendor/", + "venv/", + "astacus/proto/", +] +show_error_codes = true +warn_unreachable = true +no_implicit_reexport = true +warn_redundant_casts = true +warn_unused_ignores = true +disallow_subclassing_any = true + +[tool.pydantic-mypy] +# even if Config.extra != forbid, prevent extra args +init_forbid_extra = true +# validate types on __init__, do not go for type coercion +init_typed = true +# We don't want use of indirect population without explicit alias +warn_required_dynamic_aliases = true + +[[tool.mypy.overrides]] +module = "cassandra.*,graphlib,kazoo.*,systemd" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "google.protobuf" +ignore_errors = true + +[[tool.mypy.overrides]] +module = "astacus.common.cassandra,astacus.coordinator.cassandra,astacus.node.cassandra" +disallow_incomplete_defs = true +disallow_untyped_defs = true