Merge pull request #171 from Aiven-Open/joelynch/type-tests

tests: add typing

#171

astacus/common/rohmustorage.py

@@ -208,7 +208,7 @@ class MultiRohmuStorage(MultiStorage[RohmuStorage]):
     def __init__(self, *, config: RohmuConfig) -> None:
         self.config = config
 
-    def get_storage(self, name: str) -> RohmuStorage:
+    def get_storage(self, name: str | None) -> RohmuStorage:
         return RohmuStorage(config=self.config, storage=name)
 
     def get_default_storage_name(self) -> str:

tests/conftest.py

@@ -30,7 +30,7 @@ def pytest_addoption(parser: Parser) -> None:
     )
 
 
-def pytest_runtest_setup(item):
+def pytest_runtest_setup(item: pytest.Item) -> None:
     if any(item.iter_markers(name="x86_64")) and platform.machine() != "x86_64":
         pytest.skip("x86_64 arch required")
 

tests/integration/coordinator/plugins/clickhouse/test_plugin.py

@@ -347,7 +347,7 @@ async def test_cleanup_does_not_break_object_storage_disk_files(
     ports: Ports,
     clickhouse_command: ClickHouseCommand,
     minio_bucket: MinioBucket,
-):
+) -> None:
     with tempfile.TemporaryDirectory(prefix="storage_") as storage_path_str:
         storage_path = Path(storage_path_str)
         async with create_zookeeper(ports) as zookeeper:

tests/integration/coordinator/plugins/flink/test_steps.py

@@ -13,7 +13,7 @@
 
 
 @pytest.mark.asyncio
-async def test_restore_data(zookeeper_client: KazooZooKeeperClient):
+async def test_restore_data(zookeeper_client: KazooZooKeeperClient) -> None:
     table_id1 = str(uuid4()).partition("-")[0]
     table_id2 = str(uuid4()).partition("-")[0]
     data = {

tests/system/conftest.py

@@ -2,6 +2,7 @@
 Copyright (c) 2020 Aiven Ltd
 See LICENSE for details
 """
+from _pytest.config import Config
 from astacus.common.utils import AstacusModel, exponential_backoff
 from contextlib import asynccontextmanager
 from httpx import URL
@@ -15,6 +16,7 @@
 import json
 import logging
 import os.path
+import py
 import pytest
 import subprocess
 
@@ -134,12 +136,12 @@ async def wait_url_up(url: Union[URL, str]) -> None:
 
 
 @pytest.fixture(name="rootdir")
-def fixture_rootdir(pytestconfig) -> str:
+def fixture_rootdir(pytestconfig: Config) -> str:
     return os.path.join(os.path.dirname(__file__), "..", "..")
 
 
 @asynccontextmanager
-async def _astacus(*, tmpdir, index: int) -> AsyncIterator[TestNode]:
+async def _astacus(*, tmpdir: py.path.local, index: int) -> AsyncIterator[TestNode]:
     node = ASTACUS_NODES[index]
     a_conf_path = create_astacus_config(tmpdir=tmpdir, node=node)
     astacus_source_root = os.path.join(os.path.dirname(__file__), "..", "..")
@@ -151,19 +153,19 @@ async def _astacus(*, tmpdir, index: int) -> AsyncIterator[TestNode]:
 
 
 @pytest.fixture(name="astacus1")
-async def fixture_astacus1(tmpdir) -> AsyncIterator[TestNode]:
+async def fixture_astacus1(tmpdir: py.path.local) -> AsyncIterator[TestNode]:
     async with _astacus(tmpdir=tmpdir, index=0) as a:
         yield a
 
 
 @pytest.fixture(name="astacus2")
-async def fixture_astacus2(tmpdir) -> AsyncIterator[TestNode]:
+async def fixture_astacus2(tmpdir: py.path.local) -> AsyncIterator[TestNode]:
     async with _astacus(tmpdir=tmpdir, index=1) as a:
         yield a
 
 
 @pytest.fixture(name="astacus3")
-async def fixture_astacus3(tmpdir) -> AsyncIterator[TestNode]:
+async def fixture_astacus3(tmpdir: py.path.local) -> AsyncIterator[TestNode]:
     async with _astacus(tmpdir=tmpdir, index=2) as a:
         yield a
 

tests/system/test_astacus.py

@@ -27,7 +27,7 @@
 
 # This test is the slowest, so rather fail fast in real unittests before getting here
 @pytest.mark.order("last")
-def test_astacus(astacus1: TestNode, astacus2: TestNode, astacus3: TestNode, rootdir: str):
+def test_astacus(astacus1: TestNode, astacus2: TestNode, astacus3: TestNode, rootdir: str) -> None:
     assert astacus1.root_path
     assert astacus2.root_path
     assert astacus3.root_path

tests/unit/common/cassandra/test_schema.py

@@ -5,14 +5,15 @@
 
 from astacus.common.cassandra import schema
 from cassandra import metadata as cm
+from pytest_mock import MockerFixture
 from typing import Mapping
 
 import pytest
 
 # pylint: disable=protected-access
 
 
-def test_schema(mocker):
+def test_schema(mocker: MockerFixture) -> None:
     cut = schema.CassandraUserType(name="cut", cql_create_self="CREATE-USER-TYPE", field_types=["type1", "type2"])
     cfunction = schema.CassandraFunction(name="cf", cql_create_self="CREATE-FUNCTION", argument_types=["atype1", "atype2"])
 
@@ -26,7 +27,7 @@ def test_schema(mocker):
 
     ctrigger = schema.CassandraTrigger(name="ctrigger", cql_create_self="CREATE-TRIGGER")
 
-    cmv = schema.CassandraIndex(name="cmv", cql_create_self="CREATE-MATERIALIZED-VIEW")
+    cmv = schema.CassandraMaterializedView(name="cmv", cql_create_self="CREATE-MATERIALIZED-VIEW")
 
     ctable = schema.CassandraTable(
         name="ctable", cql_create_self="CREATE-TABLE", indexes=[cindex], materialized_views=[cmv], triggers=[ctrigger]
@@ -96,7 +97,7 @@ def test_schema_keyspace_from_metadata(
     assert keyspace.user_types == []
 
 
-def test_schema_keyspace_iterate_user_types_in_restore_order():
+def test_schema_keyspace_iterate_user_types_in_restore_order() -> None:
     ut1 = schema.CassandraUserType(name="ut1", cql_create_self="", field_types=[])
     ut2 = schema.CassandraUserType(name="ut2", cql_create_self="", field_types=["ut3", "map<str,frozen<ut1>>"])
     ut3 = schema.CassandraUserType(name="ut3", cql_create_self="", field_types=["ut4"])
@@ -131,6 +132,6 @@ def test_schema_keyspace_iterate_user_types_in_restore_order():
         ('"q""u""o""t""e""d"', ['q"u"o"t"e"d']),
     ],
 )
-def test_iterate_identifiers_in_cql_type_definition(definition, identifiers):
+def test_iterate_identifiers_in_cql_type_definition(definition: str, identifiers: list[str]) -> None:
     got_identifiers = list(schema._iterate_identifiers_in_cql_type_definition(definition))
     assert got_identifiers == identifiers

tests/unit/common/test_m3placement.py

@@ -23,7 +23,7 @@
 )
 
 
-def create_dummy_placement():
+def create_dummy_placement() -> m3_placement_pb2.Placement:
     placement = m3_placement_pb2.Placement()
     instance = placement.instances["node-id1"]
     instance.id = "node-id1"
@@ -32,7 +32,7 @@ def create_dummy_placement():
     return placement
 
 
-def test_rewrite_single_m3_placement_node():
+def test_rewrite_single_m3_placement_node() -> None:
     placement = create_dummy_placement()
     m3placement.rewrite_single_m3_placement_node(
         placement, src_pnode=src_pnode, dst_pnode=dst_pnode, dst_isolation_group="az22"
@@ -43,7 +43,7 @@ def test_rewrite_single_m3_placement_node():
     assert instance2.isolation_group == "az22"
 
 
-def test_rewrite_m3_placement_bytes():
+def test_rewrite_m3_placement_bytes() -> None:
     value = create_dummy_placement().SerializeToString()
     assert isinstance(value, bytes)
     expected_bytes = [b"endpoint22", b"hostname22", b"az22"]

tests/unit/common/test_op.py

@@ -15,19 +15,6 @@
 import pytest
 
 
-class MockOp:
-    status = None
-    op_id = 1
-
-    def set_status(self, state, *, from_status=None):
-        if from_status and from_status != self.status:
-            return
-        self.status = state
-
-    def set_status_fail(self):
-        self.status = op.Op.Status.fail
-
-
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "fun_ex,expect_status,expect_ex",
@@ -42,13 +29,15 @@ def set_status_fail(self):
     ],
 )
 @pytest.mark.parametrize("is_async", [False, True])
-async def test_opmixin_start_op(is_async, fun_ex, expect_status, expect_ex):
+async def test_opmixin_start_op(
+    is_async: bool, fun_ex: type[Exception] | None, expect_status: op.Op.Status, expect_ex: type[Exception] | None
+) -> None:
     mixin = op.OpMixin()
     mixin.state = op.OpState()
     mixin.stats = StatsClient(config=None)
     mixin.request_url = URL()
     mixin.background_tasks = BackgroundTasks()
-    op_obj = MockOp()
+    op_obj = op.Op(info=op.Op.Info(op_id=1), op_id=1, stats=StatsClient(config=None))
 
     def _sync():
         if fun_ex:
@@ -66,4 +55,4 @@ async def _async():
         await mixin.background_tasks()
     except Exception as ex:  # pylint: disable=broad-except
         assert expect_ex and isinstance(ex, expect_ex)
-    assert op_obj.status == expect_status
+    assert op_obj.info.op_status == expect_status

tests/unit/common/test_op_stats.py

@@ -40,7 +40,7 @@ class DummyStep3(DummyStep):
 
 
 @pytest.mark.asyncio
-async def test_op_stats():
+async def test_op_stats() -> None:
     stats = StatsClient(config=None)
     coordinator = Coordinator(
         request_url=URL(),
@@ -73,7 +73,7 @@ class DummyOp(op.Op):
     pass
 
 
-def test_status_fail_stats():
+def test_status_fail_stats() -> None:
     stats = StatsClient(config=None)
     operation = DummyOp(info=op.Op.Info(op_id=1, op_name="DummyOp"), op_id=1, stats=stats)
 

tests/unit/common/test_progress.py

@@ -32,12 +32,12 @@
         (16676, None, None, False),
     ],
 )
-def test_increase_worth_reporting(old_value, new_value, total, exp):
+def test_increase_worth_reporting(old_value: int, new_value: int | None, total: int | None, exp: bool) -> None:
     assert progress.increase_worth_reporting(old_value, new_value, total=total) == exp
 
 
 @pytest.mark.parametrize("is_final", [True, False])
-def test_progress_merge(is_final):
+def test_progress_merge(is_final: bool) -> None:
     p1 = progress.Progress(handled=0, failed=1000, total=10, final=True)
     p2 = progress.Progress(handled=100, failed=100, total=1000, final=is_final)
     p3 = progress.Progress(handled=1000, failed=10, total=10000, final=True)

tests/unit/common/test_statsd.py

@@ -26,9 +26,9 @@ def datagram_received(self, data: bytes, addr: tuple[str | Any, int]) -> None:
 
 
 @pytest.mark.asyncio
-async def test_statsd():
+async def test_statsd() -> None:
     loop = asyncio.get_running_loop()
-    received = asyncio.Queue()
+    received: asyncio.Queue[bytes] = asyncio.Queue()
 
     sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     sock.bind(("", 0))

tests/unit/common/test_storage.py

@@ -14,10 +14,13 @@
 from astacus.common.storage import FileStorage, Json, JsonStorage
 from contextlib import nullcontext as does_not_raise
 from pathlib import Path
+from pytest_mock import MockerFixture
 from rohmu.object_storage import google
 from tests.utils import create_rohmu_config
-from unittest.mock import patch
+from typing import ContextManager
+from unittest.mock import Mock, patch
 
+import py
 import pytest
 
 TEST_HEXDIGEST = "deadbeef"
@@ -27,7 +30,7 @@
 TEST_JSON_DATA: Json = {"foo": 7, "array": [1, 2, 3], "true": True}
 
 
-def create_storage(*, tmpdir, engine, **kw):
+def create_storage(*, tmpdir: py.path.local, engine: str, **kw):
     if engine == "rohmu":
         config = create_rohmu_config(tmpdir, **kw)
         return RohmuStorage(config=config)
@@ -43,7 +46,7 @@ def create_storage(*, tmpdir, engine, **kw):
     raise NotImplementedError(f"unknown storage {engine}")
 
 
-def _test_hexdigeststorage(storage: FileStorage):
+def _test_hexdigeststorage(storage: FileStorage) -> None:
     storage.upload_hexdigest_bytes(TEST_HEXDIGEST, TEXT_HEXDIGEST_DATA)
     assert storage.download_hexdigest_bytes(TEST_HEXDIGEST) == TEXT_HEXDIGEST_DATA
     # Ensure that download attempts of nonexistent keys give NotFoundException
@@ -56,7 +59,7 @@ def _test_hexdigeststorage(storage: FileStorage):
     assert storage.list_hexdigests() == []
 
 
-def _test_jsonstorage(storage: JsonStorage):
+def _test_jsonstorage(storage: JsonStorage) -> None:
     assert storage.list_jsons() == []
     storage.upload_json(TEST_JSON, TEST_JSON_DATA)
     assert storage.download_json(TEST_JSON) == TEST_JSON_DATA
@@ -80,7 +83,7 @@ def _test_jsonstorage(storage: JsonStorage):
         ("rohmu", {"compression": False, "encryption": False}, pytest.raises(exceptions.CompressionOrEncryptionRequired)),
     ],
 )
-def test_storage(tmpdir, engine, kw, ex):
+def test_storage(tmpdir: py.path.local, engine: str, kw: dict[str, bool], ex: ContextManager | None) -> None:
     if ex is None:
         ex = does_not_raise()
     with ex:
@@ -91,7 +94,7 @@ def test_storage(tmpdir, engine, kw, ex):
             _test_jsonstorage(storage)
 
 
-def test_caching_storage(tmpdir, mocker):
+def test_caching_storage(tmpdir: py.path.local, mocker: MockerFixture) -> None:
     storage = create_storage(tmpdir=tmpdir, engine="cache")
     storage.upload_json(TEST_JSON, TEST_JSON_DATA)
 
@@ -111,7 +114,7 @@ def test_caching_storage(tmpdir, mocker):
 
 @patch("rohmu.object_storage.google.get_credentials")
 @patch.object(google.GoogleTransfer, "_init_google_client")
-def test_proxy_storage(mock_google_client, mock_get_credentials):
+def test_proxy_storage(mock_google_client: Mock, mock_get_credentials: Mock) -> None:
     rs = RohmuStorage(
         config=RohmuConfig.parse_obj(
             {