diff --git a/astacus/common/rohmustorage.py b/astacus/common/rohmustorage.py index 41f61562..985dc838 100644 --- a/astacus/common/rohmustorage.py +++ b/astacus/common/rohmustorage.py @@ -208,7 +208,7 @@ class MultiRohmuStorage(MultiStorage[RohmuStorage]): def __init__(self, *, config: RohmuConfig) -> None: self.config = config - def get_storage(self, name: str) -> RohmuStorage: + def get_storage(self, name: str | None) -> RohmuStorage: return RohmuStorage(config=self.config, storage=name) def get_default_storage_name(self) -> str: diff --git a/tests/conftest.py b/tests/conftest.py index ea07871f..f22c0000 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,7 +30,7 @@ def pytest_addoption(parser: Parser) -> None: ) -def pytest_runtest_setup(item): +def pytest_runtest_setup(item: pytest.Item) -> None: if any(item.iter_markers(name="x86_64")) and platform.machine() != "x86_64": pytest.skip("x86_64 arch required") diff --git a/tests/integration/coordinator/plugins/clickhouse/test_plugin.py b/tests/integration/coordinator/plugins/clickhouse/test_plugin.py index eaa2b1d6..2119098f 100644 --- a/tests/integration/coordinator/plugins/clickhouse/test_plugin.py +++ b/tests/integration/coordinator/plugins/clickhouse/test_plugin.py @@ -347,7 +347,7 @@ async def test_cleanup_does_not_break_object_storage_disk_files( ports: Ports, clickhouse_command: ClickHouseCommand, minio_bucket: MinioBucket, -): +) -> None: with tempfile.TemporaryDirectory(prefix="storage_") as storage_path_str: storage_path = Path(storage_path_str) async with create_zookeeper(ports) as zookeeper: diff --git a/tests/integration/coordinator/plugins/flink/test_steps.py b/tests/integration/coordinator/plugins/flink/test_steps.py index 2a14d6e4..2db6825d 100644 --- a/tests/integration/coordinator/plugins/flink/test_steps.py +++ b/tests/integration/coordinator/plugins/flink/test_steps.py @@ -13,7 +13,7 @@ @pytest.mark.asyncio -async def test_restore_data(zookeeper_client: KazooZooKeeperClient): +async def test_restore_data(zookeeper_client: KazooZooKeeperClient) -> None: table_id1 = str(uuid4()).partition("-")[0] table_id2 = str(uuid4()).partition("-")[0] data = { diff --git a/tests/system/conftest.py b/tests/system/conftest.py index 947c80ab..1ef0c7b4 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -2,6 +2,7 @@ Copyright (c) 2020 Aiven Ltd See LICENSE for details """ +from _pytest.config import Config from astacus.common.utils import AstacusModel, exponential_backoff from contextlib import asynccontextmanager from httpx import URL @@ -15,6 +16,7 @@ import json import logging import os.path +import py import pytest import subprocess @@ -134,12 +136,12 @@ async def wait_url_up(url: Union[URL, str]) -> None: @pytest.fixture(name="rootdir") -def fixture_rootdir(pytestconfig) -> str: +def fixture_rootdir(pytestconfig: Config) -> str: return os.path.join(os.path.dirname(__file__), "..", "..") @asynccontextmanager -async def _astacus(*, tmpdir, index: int) -> AsyncIterator[TestNode]: +async def _astacus(*, tmpdir: py.path.local, index: int) -> AsyncIterator[TestNode]: node = ASTACUS_NODES[index] a_conf_path = create_astacus_config(tmpdir=tmpdir, node=node) astacus_source_root = os.path.join(os.path.dirname(__file__), "..", "..") @@ -151,19 +153,19 @@ async def _astacus(*, tmpdir, index: int) -> AsyncIterator[TestNode]: @pytest.fixture(name="astacus1") -async def fixture_astacus1(tmpdir) -> AsyncIterator[TestNode]: +async def fixture_astacus1(tmpdir: py.path.local) -> AsyncIterator[TestNode]: async with _astacus(tmpdir=tmpdir, index=0) as a: yield a @pytest.fixture(name="astacus2") -async def fixture_astacus2(tmpdir) -> AsyncIterator[TestNode]: +async def fixture_astacus2(tmpdir: py.path.local) -> AsyncIterator[TestNode]: async with _astacus(tmpdir=tmpdir, index=1) as a: yield a @pytest.fixture(name="astacus3") -async def fixture_astacus3(tmpdir) -> AsyncIterator[TestNode]: +async def fixture_astacus3(tmpdir: py.path.local) -> AsyncIterator[TestNode]: async with _astacus(tmpdir=tmpdir, index=2) as a: yield a diff --git a/tests/system/test_astacus.py b/tests/system/test_astacus.py index 980ba2ae..a5242981 100644 --- a/tests/system/test_astacus.py +++ b/tests/system/test_astacus.py @@ -27,7 +27,7 @@ # This test is the slowest, so rather fail fast in real unittests before getting here @pytest.mark.order("last") -def test_astacus(astacus1: TestNode, astacus2: TestNode, astacus3: TestNode, rootdir: str): +def test_astacus(astacus1: TestNode, astacus2: TestNode, astacus3: TestNode, rootdir: str) -> None: assert astacus1.root_path assert astacus2.root_path assert astacus3.root_path diff --git a/tests/unit/common/cassandra/test_schema.py b/tests/unit/common/cassandra/test_schema.py index 3ab6aa04..c0e1aed7 100644 --- a/tests/unit/common/cassandra/test_schema.py +++ b/tests/unit/common/cassandra/test_schema.py @@ -5,6 +5,7 @@ from astacus.common.cassandra import schema from cassandra import metadata as cm +from pytest_mock import MockerFixture from typing import Mapping import pytest @@ -12,7 +13,7 @@ # pylint: disable=protected-access -def test_schema(mocker): +def test_schema(mocker: MockerFixture) -> None: cut = schema.CassandraUserType(name="cut", cql_create_self="CREATE-USER-TYPE", field_types=["type1", "type2"]) cfunction = schema.CassandraFunction(name="cf", cql_create_self="CREATE-FUNCTION", argument_types=["atype1", "atype2"]) @@ -26,7 +27,7 @@ def test_schema(mocker): ctrigger = schema.CassandraTrigger(name="ctrigger", cql_create_self="CREATE-TRIGGER") - cmv = schema.CassandraIndex(name="cmv", cql_create_self="CREATE-MATERIALIZED-VIEW") + cmv = schema.CassandraMaterializedView(name="cmv", cql_create_self="CREATE-MATERIALIZED-VIEW") ctable = schema.CassandraTable( name="ctable", cql_create_self="CREATE-TABLE", indexes=[cindex], materialized_views=[cmv], triggers=[ctrigger] @@ -96,7 +97,7 @@ def test_schema_keyspace_from_metadata( assert keyspace.user_types == [] -def test_schema_keyspace_iterate_user_types_in_restore_order(): +def test_schema_keyspace_iterate_user_types_in_restore_order() -> None: ut1 = schema.CassandraUserType(name="ut1", cql_create_self="", field_types=[]) ut2 = schema.CassandraUserType(name="ut2", cql_create_self="", field_types=["ut3", "map>"]) ut3 = schema.CassandraUserType(name="ut3", cql_create_self="", field_types=["ut4"]) @@ -131,6 +132,6 @@ def test_schema_keyspace_iterate_user_types_in_restore_order(): ('"q""u""o""t""e""d"', ['q"u"o"t"e"d']), ], ) -def test_iterate_identifiers_in_cql_type_definition(definition, identifiers): +def test_iterate_identifiers_in_cql_type_definition(definition: str, identifiers: list[str]) -> None: got_identifiers = list(schema._iterate_identifiers_in_cql_type_definition(definition)) assert got_identifiers == identifiers diff --git a/tests/unit/common/test_m3placement.py b/tests/unit/common/test_m3placement.py index 94ce49bf..0e529023 100644 --- a/tests/unit/common/test_m3placement.py +++ b/tests/unit/common/test_m3placement.py @@ -23,7 +23,7 @@ ) -def create_dummy_placement(): +def create_dummy_placement() -> m3_placement_pb2.Placement: placement = m3_placement_pb2.Placement() instance = placement.instances["node-id1"] instance.id = "node-id1" @@ -32,7 +32,7 @@ def create_dummy_placement(): return placement -def test_rewrite_single_m3_placement_node(): +def test_rewrite_single_m3_placement_node() -> None: placement = create_dummy_placement() m3placement.rewrite_single_m3_placement_node( placement, src_pnode=src_pnode, dst_pnode=dst_pnode, dst_isolation_group="az22" @@ -43,7 +43,7 @@ def test_rewrite_single_m3_placement_node(): assert instance2.isolation_group == "az22" -def test_rewrite_m3_placement_bytes(): +def test_rewrite_m3_placement_bytes() -> None: value = create_dummy_placement().SerializeToString() assert isinstance(value, bytes) expected_bytes = [b"endpoint22", b"hostname22", b"az22"] diff --git a/tests/unit/common/test_op.py b/tests/unit/common/test_op.py index 1bdf4b0f..7241286f 100644 --- a/tests/unit/common/test_op.py +++ b/tests/unit/common/test_op.py @@ -15,19 +15,6 @@ import pytest -class MockOp: - status = None - op_id = 1 - - def set_status(self, state, *, from_status=None): - if from_status and from_status != self.status: - return - self.status = state - - def set_status_fail(self): - self.status = op.Op.Status.fail - - @pytest.mark.asyncio @pytest.mark.parametrize( "fun_ex,expect_status,expect_ex", @@ -42,13 +29,15 @@ def set_status_fail(self): ], ) @pytest.mark.parametrize("is_async", [False, True]) -async def test_opmixin_start_op(is_async, fun_ex, expect_status, expect_ex): +async def test_opmixin_start_op( + is_async: bool, fun_ex: type[Exception] | None, expect_status: op.Op.Status, expect_ex: type[Exception] | None +) -> None: mixin = op.OpMixin() mixin.state = op.OpState() mixin.stats = StatsClient(config=None) mixin.request_url = URL() mixin.background_tasks = BackgroundTasks() - op_obj = MockOp() + op_obj = op.Op(info=op.Op.Info(op_id=1), op_id=1, stats=StatsClient(config=None)) def _sync(): if fun_ex: @@ -66,4 +55,4 @@ async def _async(): await mixin.background_tasks() except Exception as ex: # pylint: disable=broad-except assert expect_ex and isinstance(ex, expect_ex) - assert op_obj.status == expect_status + assert op_obj.info.op_status == expect_status diff --git a/tests/unit/common/test_op_stats.py b/tests/unit/common/test_op_stats.py index 6e116429..2fe3ed57 100644 --- a/tests/unit/common/test_op_stats.py +++ b/tests/unit/common/test_op_stats.py @@ -40,7 +40,7 @@ class DummyStep3(DummyStep): @pytest.mark.asyncio -async def test_op_stats(): +async def test_op_stats() -> None: stats = StatsClient(config=None) coordinator = Coordinator( request_url=URL(), @@ -73,7 +73,7 @@ class DummyOp(op.Op): pass -def test_status_fail_stats(): +def test_status_fail_stats() -> None: stats = StatsClient(config=None) operation = DummyOp(info=op.Op.Info(op_id=1, op_name="DummyOp"), op_id=1, stats=stats) diff --git a/tests/unit/common/test_progress.py b/tests/unit/common/test_progress.py index 4001f6b4..598238b2 100644 --- a/tests/unit/common/test_progress.py +++ b/tests/unit/common/test_progress.py @@ -32,12 +32,12 @@ (16676, None, None, False), ], ) -def test_increase_worth_reporting(old_value, new_value, total, exp): +def test_increase_worth_reporting(old_value: int, new_value: int | None, total: int | None, exp: bool) -> None: assert progress.increase_worth_reporting(old_value, new_value, total=total) == exp @pytest.mark.parametrize("is_final", [True, False]) -def test_progress_merge(is_final): +def test_progress_merge(is_final: bool) -> None: p1 = progress.Progress(handled=0, failed=1000, total=10, final=True) p2 = progress.Progress(handled=100, failed=100, total=1000, final=is_final) p3 = progress.Progress(handled=1000, failed=10, total=10000, final=True) diff --git a/tests/unit/common/test_statsd.py b/tests/unit/common/test_statsd.py index 839b2e29..da8097eb 100644 --- a/tests/unit/common/test_statsd.py +++ b/tests/unit/common/test_statsd.py @@ -26,9 +26,9 @@ def datagram_received(self, data: bytes, addr: tuple[str | Any, int]) -> None: @pytest.mark.asyncio -async def test_statsd(): +async def test_statsd() -> None: loop = asyncio.get_running_loop() - received = asyncio.Queue() + received: asyncio.Queue[bytes] = asyncio.Queue() sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.bind(("", 0)) diff --git a/tests/unit/common/test_storage.py b/tests/unit/common/test_storage.py index 8bbf46c5..165120fa 100644 --- a/tests/unit/common/test_storage.py +++ b/tests/unit/common/test_storage.py @@ -14,10 +14,13 @@ from astacus.common.storage import FileStorage, Json, JsonStorage from contextlib import nullcontext as does_not_raise from pathlib import Path +from pytest_mock import MockerFixture from rohmu.object_storage import google from tests.utils import create_rohmu_config -from unittest.mock import patch +from typing import ContextManager +from unittest.mock import Mock, patch +import py import pytest TEST_HEXDIGEST = "deadbeef" @@ -27,7 +30,7 @@ TEST_JSON_DATA: Json = {"foo": 7, "array": [1, 2, 3], "true": True} -def create_storage(*, tmpdir, engine, **kw): +def create_storage(*, tmpdir: py.path.local, engine: str, **kw): if engine == "rohmu": config = create_rohmu_config(tmpdir, **kw) return RohmuStorage(config=config) @@ -43,7 +46,7 @@ def create_storage(*, tmpdir, engine, **kw): raise NotImplementedError(f"unknown storage {engine}") -def _test_hexdigeststorage(storage: FileStorage): +def _test_hexdigeststorage(storage: FileStorage) -> None: storage.upload_hexdigest_bytes(TEST_HEXDIGEST, TEXT_HEXDIGEST_DATA) assert storage.download_hexdigest_bytes(TEST_HEXDIGEST) == TEXT_HEXDIGEST_DATA # Ensure that download attempts of nonexistent keys give NotFoundException @@ -56,7 +59,7 @@ def _test_hexdigeststorage(storage: FileStorage): assert storage.list_hexdigests() == [] -def _test_jsonstorage(storage: JsonStorage): +def _test_jsonstorage(storage: JsonStorage) -> None: assert storage.list_jsons() == [] storage.upload_json(TEST_JSON, TEST_JSON_DATA) assert storage.download_json(TEST_JSON) == TEST_JSON_DATA @@ -80,7 +83,7 @@ def _test_jsonstorage(storage: JsonStorage): ("rohmu", {"compression": False, "encryption": False}, pytest.raises(exceptions.CompressionOrEncryptionRequired)), ], ) -def test_storage(tmpdir, engine, kw, ex): +def test_storage(tmpdir: py.path.local, engine: str, kw: dict[str, bool], ex: ContextManager | None) -> None: if ex is None: ex = does_not_raise() with ex: @@ -91,7 +94,7 @@ def test_storage(tmpdir, engine, kw, ex): _test_jsonstorage(storage) -def test_caching_storage(tmpdir, mocker): +def test_caching_storage(tmpdir: py.path.local, mocker: MockerFixture) -> None: storage = create_storage(tmpdir=tmpdir, engine="cache") storage.upload_json(TEST_JSON, TEST_JSON_DATA) @@ -111,7 +114,7 @@ def test_caching_storage(tmpdir, mocker): @patch("rohmu.object_storage.google.get_credentials") @patch.object(google.GoogleTransfer, "_init_google_client") -def test_proxy_storage(mock_google_client, mock_get_credentials): +def test_proxy_storage(mock_google_client: Mock, mock_get_credentials: Mock) -> None: rs = RohmuStorage( config=RohmuConfig.parse_obj( { diff --git a/tests/unit/common/test_utils.py b/tests/unit/common/test_utils.py index 59c7e266..cabc98ea 100644 --- a/tests/unit/common/test_utils.py +++ b/tests/unit/common/test_utils.py @@ -10,9 +10,11 @@ from astacus.common.utils import AsyncSleeper, build_netloc, parse_umask from datetime import timedelta from pathlib import Path +from pytest_mock import MockerFixture import asyncio import logging +import py import pytest import tempfile import time @@ -52,7 +54,7 @@ async def test_httpx_request_connect_failure(): @pytest.mark.asyncio -async def test_async_sleeper(): +async def test_async_sleeper() -> None: sleeper = AsyncSleeper() async def wait_and_wake(): @@ -69,8 +71,8 @@ async def wait_and_wake(): @pytest.mark.asyncio -async def test_exponential_backoff(mocker): - _waits = [] +async def test_exponential_backoff(mocker: MockerFixture) -> None: + _waits: list[float] = [] base = 42 def _time_monotonic(): @@ -115,7 +117,7 @@ def _assert_rounded_waits_equals(x): (timedelta(seconds=0), ""), ], ) -def test_timedelta_as_short_str(v, s): +def test_timedelta_as_short_str(v: timedelta, s: str) -> None: assert utils.timedelta_as_short_str(v) == s @@ -130,11 +132,11 @@ def test_timedelta_as_short_str(v, s): (1, "1 B"), ], ) -def test_size_as_short_str(v, s): +def test_size_as_short_str(v: int, s: str) -> None: assert utils.size_as_short_str(v) == s -def test_sizelimitedfile(): +def test_sizelimitedfile() -> None: with tempfile.NamedTemporaryFile() as f: f.write(b"foobarbaz") f.flush() @@ -150,7 +152,7 @@ def test_sizelimitedfile(): assert lf.read() == b"bar" -def test_open_path_with_atomic_rename(tmpdir): +def test_open_path_with_atomic_rename(tmpdir: py.path.local) -> None: # default is bytes f1_path = f"{tmpdir}/f1" with utils.open_path_with_atomic_rename(f1_path) as f1: @@ -170,10 +172,15 @@ def test_open_path_with_atomic_rename(tmpdir): # erroneous cases should not produce file at all f3_path = f"{tmpdir}/f3" - with pytest.raises(AssertionError): + + class TestException(RuntimeError): + pass + + with pytest.raises(TestException): with utils.open_path_with_atomic_rename(f3_path): - assert False - assert not Path(f3_path).exists() + raise TestException() + # This version of MyPy works poorly with pytest.raises + assert not Path(f3_path).exists() # type: ignore[unreachable] def test_parse_umask() -> None: diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index b2d640f4..b38f9898 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -3,45 +3,21 @@ See LICENSE for details """ -from astacus.common import utils from astacus.common.cassandra.config import CassandraClientConfiguration, SNAPSHOT_NAME from io import StringIO from pathlib import Path +from pytest_mock import MockerFixture import logging +import py import pytest import tempfile logger = logging.getLogger(__name__) -@pytest.fixture(name="utils_http_request_list") -def fixture_utils_http_request_list(mocker): - result_list = [] - - def _http_request(*args, **kwargs): - logger.info("utils_http_request_list %r %r", args, kwargs) - assert result_list, f"Unable to serve request {args!r} {kwargs!r}" - r = result_list.pop(0) - if isinstance(r, dict): - if "args" in r: - assert list(r["args"]) == list(args) - if "kwargs" in r: - assert r["kwargs"] == kwargs - if "result" in r: - return r["result"] - return None - if r is None: - return None - raise NotImplementedError(f"Unknown item in utils_request_list: {r!r}") - - mocker.patch.object(utils, "http_request", new=_http_request) - yield result_list - assert not result_list, f"Unconsumed requests: {result_list!r}" - - class CassandraTestConfig: - def __init__(self, *, mocker, tmpdir): + def __init__(self, *, mocker: MockerFixture, tmpdir: py.path.local): self.root = Path(tmpdir) / "root" # Create fake snapshot files that we want to see being moved/removed/.. @@ -87,5 +63,5 @@ def __init__(self, *, mocker, tmpdir): @pytest.fixture(name="cassandra_test_config") -def fixture_cassandra_test_config(mocker, tmpdir): - yield CassandraTestConfig(mocker=mocker, tmpdir=tmpdir) +def fixture_cassandra_test_config(mocker: MockerFixture, tmpdir: py.path.local) -> CassandraTestConfig: + return CassandraTestConfig(mocker=mocker, tmpdir=tmpdir) diff --git a/tests/unit/coordinator/conftest.py b/tests/unit/coordinator/conftest.py index 40772b37..129f63a6 100644 --- a/tests/unit/coordinator/conftest.py +++ b/tests/unit/coordinator/conftest.py @@ -3,27 +3,31 @@ See LICENSE for details """ from .test_restore import BACKUP_MANIFEST +from astacus.common.ipc import Plugin from astacus.common.rohmustorage import MultiRohmuStorage, RohmuStorage from astacus.coordinator.api import router from astacus.coordinator.config import CoordinatorConfig, CoordinatorNode from astacus.coordinator.coordinator import LockedCoordinatorOp from fastapi import FastAPI from fastapi.testclient import TestClient +from pathlib import Path +from pytest_mock import MockerFixture from tests.utils import create_rohmu_config import asyncio +import py import pytest _original_asyncio_sleep = asyncio.sleep @pytest.fixture(name="storage") -def fixture_storage(tmpdir): +def fixture_storage(tmpdir: py.path.local) -> RohmuStorage: return RohmuStorage(config=create_rohmu_config(tmpdir)) @pytest.fixture(name="mstorage") -def fixture_mstorage(tmpdir): +def fixture_mstorage(tmpdir: py.path.local) -> MultiRohmuStorage: return MultiRohmuStorage(config=create_rohmu_config(tmpdir)) @@ -40,22 +44,22 @@ def fixture_populated_mstorage(mstorage: MultiRohmuStorage) -> MultiRohmuStorage @pytest.fixture(name="client") -def fixture_client(app): +def fixture_client(app: FastAPI) -> TestClient: client = TestClient(app) # One ping at API to populate the fixtures (ugh) response = client.post("/lock") assert response.status_code == 422, response.json() - yield client + return client -async def _sleep_little(value): +async def _sleep_little(value: float) -> None: await _original_asyncio_sleep(min(value, 0.01)) @pytest.fixture(name="sleepless") -def fixture_sleepless(mocker): +def fixture_sleepless(mocker: MockerFixture) -> None: mocker.patch.object(asyncio, "sleep", new=_sleep_little) @@ -66,15 +70,15 @@ def fixture_sleepless(mocker): @pytest.fixture(name="app") -def fixture_app(mocker, sleepless, storage, tmpdir): +def fixture_app(mocker: MockerFixture, sleepless: None, storage: RohmuStorage, tmpdir: py.path.local) -> FastAPI: app = FastAPI() app.include_router(router, tags=["coordinator"]) app.state.coordinator_config = CoordinatorConfig( object_storage=create_rohmu_config(tmpdir), - plugin="files", + plugin=Plugin.files, plugin_config={"root_globs": ["*"]}, - object_storage_cache=f"{tmpdir}/cache/is/somewhere", + object_storage_cache=Path(f"{tmpdir}/cache/is/somewhere"), ) app.state.coordinator_config.nodes = COORDINATOR_NODES[:] mocker.patch.object(LockedCoordinatorOp, "get_locker", return_value="x") - yield app + return app diff --git a/tests/unit/coordinator/plugins/cassandra/test_backup_steps.py b/tests/unit/coordinator/plugins/cassandra/test_backup_steps.py index eae089d8..1e100a50 100644 --- a/tests/unit/coordinator/plugins/cassandra/test_backup_steps.py +++ b/tests/unit/coordinator/plugins/cassandra/test_backup_steps.py @@ -9,9 +9,10 @@ from astacus.coordinator.plugins.cassandra import backup_steps from astacus.coordinator.plugins.cassandra.model import CassandraConfigurationNode from dataclasses import dataclass +from pytest_mock import MockerFixture from tests.unit.coordinator.plugins.cassandra.builders import build_keyspace -from types import SimpleNamespace from typing import Mapping, Optional +from unittest.mock import Mock from uuid import UUID import pytest @@ -45,10 +46,10 @@ def __str__(self): + [RetrieveTestCase(name=f"match by {field}", field=field) for field in ["address", "host_id", "listen_address"]], ids=str, ) -def test_retrieve_manifest_from_cassandra(mocker, case): +def test_retrieve_manifest_from_cassandra(mocker: MockerFixture, case: RetrieveTestCase) -> None: mocker.patch.object(CassandraSchema, "from_cassandra_session", return_value=CassandraSchema(keyspaces=[])) cassandra_nodes = [ - SimpleNamespace( + Mock( host_id=UUID(f"1234567812345678123456781234567{node}"), address=f"1.2.3.{node}" if not case.duplicate_address else "127.0.0.1", listen_address="la" if node == 0 else None, @@ -58,14 +59,14 @@ def test_retrieve_manifest_from_cassandra(mocker, case): for node in range(case.populate_nodes) ] token_to_host_owner_map_items = [ - (SimpleNamespace(value=f"token{token}"), cassandra_nodes[node]) + (Mock(value=f"token{token}"), cassandra_nodes[node]) for node in range(case.populate_nodes) for token in range(case.populate_tokens) if ((node + node * token) // case.populate_rf) % case.populate_nodes == 0 ] - mocked_map = SimpleNamespace(items=mocker.Mock(return_value=token_to_host_owner_map_items)) - cas = SimpleNamespace(cluster_metadata=SimpleNamespace(token_map=SimpleNamespace(token_to_host_owner=mocked_map))) + mocked_map = Mock(items=mocker.Mock(return_value=token_to_host_owner_map_items)) + cas = Mock(cluster_metadata=Mock(token_map=Mock(token_to_host_owner=mocked_map))) nodes = [] for cassandra_node in cassandra_nodes: diff --git a/tests/unit/coordinator/plugins/cassandra/test_plugin.py b/tests/unit/coordinator/plugins/cassandra/test_plugin.py index f93c0e93..81140091 100644 --- a/tests/unit/coordinator/plugins/cassandra/test_plugin.py +++ b/tests/unit/coordinator/plugins/cassandra/test_plugin.py @@ -4,55 +4,55 @@ """ from astacus.common import ipc -from astacus.coordinator.plugins.base import StepFailedError +from astacus.coordinator.cluster import Cluster +from astacus.coordinator.plugins.base import StepFailedError, StepsContext from astacus.coordinator.plugins.cassandra import plugin from astacus.coordinator.plugins.cassandra.model import CassandraConfigurationNode +from pytest_mock import MockerFixture from tests.unit.conftest import CassandraTestConfig -from types import SimpleNamespace +from unittest.mock import Mock import pytest @pytest.fixture(name="cplugin") -def fixture_cplugin(mocker, tmpdir): - ctc = CassandraTestConfig(mocker=mocker, tmpdir=tmpdir) - yield plugin.CassandraPlugin( - client=ctc.cassandra_client_config, nodes=[CassandraConfigurationNode(listen_address="127.0.0.1")] +def fixture_cplugin(cassandra_test_config: CassandraTestConfig) -> plugin.CassandraPlugin: + return plugin.CassandraPlugin( + client=cassandra_test_config.cassandra_client_config, + nodes=[CassandraConfigurationNode(listen_address="127.0.0.1")], ) @pytest.mark.asyncio -async def test_step_cassandrasubop(mocker): +async def test_step_cassandrasubop(mocker: MockerFixture) -> None: mocker.patch.object(plugin, "run_subop") step = plugin.CassandraSubOpStep(op=ipc.CassandraSubOp.stop_cassandra) - cluster = None - context = None - result = await step.run_step(cluster, context) - assert result is None + cluster = Cluster(nodes=[]) + context = StepsContext() + await step.run_step(cluster, context) @pytest.mark.parametrize("success", [False, True]) @pytest.mark.asyncio -async def test_step_cassandra_validate_configuration(mocker, success): +async def test_step_cassandra_validate_configuration(mocker: MockerFixture, success: bool) -> None: step = plugin.ValidateConfigurationStep(nodes=[]) - context = None + context = Mock() if success: - cluster = SimpleNamespace(nodes=[]) - result = await step.run_step(cluster, context) - assert result is None + cluster = Mock(nodes=[]) + await step.run_step(cluster, context) else: # node count mismatch - cluster = SimpleNamespace(nodes=[42]) + cluster = Mock(nodes=[42]) with pytest.raises(StepFailedError): await step.run_step(cluster, context) -def test_get_backup_steps(mocker, cplugin): +def test_get_backup_steps(mocker: MockerFixture, cplugin): context = mocker.Mock() cplugin.get_backup_steps(context=context) -def test_get_restore_steps(mocker, cplugin): +def test_get_restore_steps(mocker: MockerFixture, cplugin): context = mocker.Mock() cplugin.get_restore_steps(context=context, req=ipc.RestoreRequest()) diff --git a/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py b/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py index 1b6e729c..a12d05d1 100644 --- a/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py +++ b/tests/unit/coordinator/plugins/cassandra/test_restore_steps.py @@ -5,12 +5,14 @@ from astacus.common import ipc from astacus.common.cassandra.schema import CassandraSchema +from astacus.coordinator.cluster import Cluster from astacus.coordinator.config import CoordinatorNode from astacus.coordinator.plugins import base from astacus.coordinator.plugins.cassandra import restore_steps from astacus.coordinator.plugins.cassandra.model import CassandraConfigurationNode, CassandraManifest, CassandraManifestNode +from pytest_mock import MockerFixture from tests.unit.coordinator.plugins.cassandra.builders import build_keyspace -from types import SimpleNamespace +from unittest.mock import Mock from uuid import UUID import datetime @@ -51,7 +53,7 @@ def _coordinator_node(node_index: int) -> CoordinatorNode: @pytest.mark.parametrize("override_tokens", [False, True]) @pytest.mark.parametrize("replace_backup_nodes", [False, True]) @pytest.mark.asyncio -async def test_step_start_cassandra(mocker, override_tokens, replace_backup_nodes): +async def test_step_start_cassandra(mocker: MockerFixture, override_tokens: bool, replace_backup_nodes: bool) -> None: plugin_manifest = CassandraManifest( cassandra_schema=CassandraSchema(keyspaces=[]), nodes=[_manifest_node(1)], @@ -93,10 +95,10 @@ def get_result(cl): step = restore_steps.StartCassandraStep( replace_backup_nodes=replace_backup_nodes, override_tokens=override_tokens, cassandra_nodes=[_configuration_node(1)] ) - context = SimpleNamespace(get_result=get_result) - cluster = SimpleNamespace(nodes=nodes) - result = await step.run_step(cluster, context) - assert result is None + context = base.StepsContext() + mocker.patch.object(context, "get_result", new=get_result) + cluster = Cluster(nodes=nodes) + await step.run_step(cluster, context) run_subop.assert_awaited_once_with( cluster, ipc.CassandraSubOp.start_cassandra, @@ -107,7 +109,7 @@ def get_result(cl): @pytest.mark.asyncio -async def test_step_stop_replaced_nodes(mocker): +async def test_step_stop_replaced_nodes(mocker: MockerFixture) -> None: # Node 3 is replacing node 1. manifest_nodes = [_manifest_node(1), _manifest_node(2)] cassandra_nodes = [_configuration_node(1), _configuration_node(2), _configuration_node(3)] @@ -129,8 +131,8 @@ def get_result(cl): run_subop = mocker.patch.object(restore_steps, "run_subop") step = restore_steps.StopReplacedNodesStep(partial_restore_nodes=partial_restore_nodes, cassandra_nodes=cassandra_nodes) - context = SimpleNamespace(get_result=get_result) - cluster = SimpleNamespace(nodes=nodes) + context = Mock(get_result=get_result) + cluster = Mock(nodes=nodes) result = await step.run_step(cluster, context) assert result == [_coordinator_node(1)] run_subop.assert_awaited_once_with( @@ -158,7 +160,7 @@ async def __anext__(self): @pytest.mark.parametrize("steps,success", [([True], True), ([False, True], True), ([False], False)]) @pytest.mark.asyncio -async def test_step_wait_cassandra_up(mocker, steps, success): +async def test_step_wait_cassandra_up(mocker: MockerFixture, steps: list[bool], success: bool) -> None: get_schema_steps = steps[:] async def get_schema_hash(cluster): @@ -167,14 +169,17 @@ async def get_schema_hash(cluster): mocker.patch.object(restore_steps, "get_schema_hash", new=get_schema_hash) - mocker.patch.object(restore_steps.utils, "exponential_backoff", return_value=AsyncIterableWrapper(steps)) + mocker.patch.object( + restore_steps.utils, # type: ignore[attr-defined] + "exponential_backoff", + return_value=AsyncIterableWrapper(steps), + ) step = restore_steps.WaitCassandraUpStep(duration=123) - context = None - cluster = None + cluster = Cluster(nodes=[]) + context = base.StepsContext() if success: - result = await step.run_step(cluster, context) - assert result is None + await step.run_step(cluster, context) else: with pytest.raises(base.StepFailedError): await step.run_step(cluster, context) diff --git a/tests/unit/coordinator/plugins/cassandra/test_utils.py b/tests/unit/coordinator/plugins/cassandra/test_utils.py index b32aee4d..bfa69714 100644 --- a/tests/unit/coordinator/plugins/cassandra/test_utils.py +++ b/tests/unit/coordinator/plugins/cassandra/test_utils.py @@ -6,14 +6,15 @@ from astacus.common import ipc from astacus.coordinator.plugins.base import StepFailedError from astacus.coordinator.plugins.cassandra import utils -from types import SimpleNamespace +from pytest_mock import MockerFixture +from unittest.mock import Mock import pytest @pytest.mark.parametrize("start_ok", [False, True]) @pytest.mark.asyncio -async def test_run_subop(mocker, start_ok): +async def test_run_subop(mocker: MockerFixture, start_ok: bool) -> None: async def request_from_nodes(*args, **kwargs): if start_ok: return 42 @@ -23,7 +24,7 @@ async def wait_successful_results(*, start_results, result_class): assert start_results == 42 return 7 - cluster = SimpleNamespace(request_from_nodes=request_from_nodes, wait_successful_results=wait_successful_results) + cluster = Mock(request_from_nodes=request_from_nodes, wait_successful_results=wait_successful_results) try: result = await utils.run_subop(cluster=cluster, subop=ipc.CassandraSubOp.stop_cassandra, result_class=ipc.NodeResult) except StepFailedError: @@ -42,7 +43,7 @@ async def wait_successful_results(*, start_results, result_class): ], ) @pytest.mark.asyncio -async def test_get_schema_hash(mocker, hashes, result): - mocker.patch.object(utils, "run_subop", return_value=[SimpleNamespace(schema_hash=hash) for hash in hashes]) - actual_result = await utils.get_schema_hash(None) +async def test_get_schema_hash(mocker: MockerFixture, hashes: list[int], result: tuple[str, str]) -> None: + mocker.patch.object(utils, "run_subop", return_value=[Mock(schema_hash=hash) for hash in hashes]) + actual_result = await utils.get_schema_hash(mocker.Mock()) assert actual_result == result diff --git a/tests/unit/coordinator/plugins/test_m3db.py b/tests/unit/coordinator/plugins/test_m3db.py index 2378211f..f1cb0239 100644 --- a/tests/unit/coordinator/plugins/test_m3db.py +++ b/tests/unit/coordinator/plugins/test_m3db.py @@ -135,7 +135,7 @@ class RestoreTest: @pytest.mark.asyncio @pytest.mark.parametrize("rt", [RestoreTest(fail_at=i) for i in range(3)] + [RestoreTest()]) -async def test_m3_restore(coordinator: Coordinator, plugin: M3DBPlugin, etcd_client: ETCDClient, rt: RestoreTest): +async def test_m3_restore(coordinator: Coordinator, plugin: M3DBPlugin, etcd_client: ETCDClient, rt: RestoreTest) -> None: partial_restore_nodes: Optional[List[ipc.PartialRestoreRequestNode]] = None if rt.partial: partial_restore_nodes = [ipc.PartialRestoreRequestNode(backup_index=0, node_index=0)] diff --git a/tests/unit/coordinator/plugins/test_zookeeper.py b/tests/unit/coordinator/plugins/test_zookeeper.py index d3abd382..c22b5493 100644 --- a/tests/unit/coordinator/plugins/test_zookeeper.py +++ b/tests/unit/coordinator/plugins/test_zookeeper.py @@ -173,7 +173,7 @@ async def test_fake_zookeeper_client_get_children_watch() -> None: @pytest.mark.asyncio -async def test_fake_zookeeper_transaction(): +async def test_fake_zookeeper_transaction() -> None: client = FakeZooKeeperClient() async with client.connect() as connection: transaction = connection.transaction() @@ -185,7 +185,7 @@ async def test_fake_zookeeper_transaction(): @pytest.mark.asyncio -async def test_fake_zookeeper_transaction_is_atomic(): +async def test_fake_zookeeper_transaction_is_atomic() -> None: client = FakeZooKeeperClient() async with client.connect() as connection: await connection.create("/key_2", b"old content") @@ -203,7 +203,7 @@ async def test_fake_zookeeper_transaction_is_atomic(): @pytest.mark.asyncio -async def test_fake_zookeeper_transaction_does_not_implicitly_create_parents(): +async def test_fake_zookeeper_transaction_does_not_implicitly_create_parents() -> None: client = FakeZooKeeperClient() async with client.connect() as connection: transaction = connection.transaction() @@ -213,7 +213,7 @@ async def test_fake_zookeeper_transaction_does_not_implicitly_create_parents(): @pytest.mark.asyncio -async def test_fake_zookeeper_transaction_generates_trigger(): +async def test_fake_zookeeper_transaction_generates_trigger() -> None: client = FakeZooKeeperClient() change_watch = ChangeWatch() async with client.connect() as connection: diff --git a/tests/unit/coordinator/test_backup.py b/tests/unit/coordinator/test_backup.py index ba229961..0c1da648 100644 --- a/tests/unit/coordinator/test_backup.py +++ b/tests/unit/coordinator/test_backup.py @@ -9,11 +9,14 @@ from astacus.common import ipc, utils from astacus.common.ipc import SnapshotHash from astacus.common.progress import Progress +from astacus.common.rohmustorage import RohmuStorage from astacus.common.statsd import StatsClient from astacus.coordinator.api import OpName from astacus.coordinator.plugins.base import build_node_index_datas, NodeIndexData from astacus.node.api import metadata -from unittest.mock import patch +from fastapi import FastAPI +from fastapi.testclient import TestClient +from unittest.mock import Mock, patch import itertools import pytest @@ -23,7 +26,7 @@ @pytest.mark.parametrize("fail_at", FAILS) -def test_backup(fail_at, app, client, storage): +def test_backup(fail_at: int | None, app: FastAPI, client: TestClient, storage: RohmuStorage) -> None: nodes = app.state.coordinator_config.nodes with respx.mock: for node in nodes: @@ -96,7 +99,7 @@ def test_backup(fail_at, app, client, storage): _progress_done = Progress(final=True) -def _ssresults(*kwarg_list): +def _ssresults(*kwarg_list) -> list[ipc.SnapshotResult]: return [ ipc.SnapshotResult(progress=_progress_done, hostname="host-{i}", start=utils.now(), **kw) for i, kw in enumerate(kwarg_list, 1) @@ -153,12 +156,16 @@ def _ssresults(*kwarg_list): ), ], ) -def test_upload_optimization(hexdigests, snapshot_results, uploads): - assert uploads == build_node_index_datas(hexdigests=hexdigests, snapshots=snapshot_results, node_indices=[0, 1, 2, 3]) +def test_upload_optimization( + hexdigests: list[str], snapshot_results: list[ipc.SnapshotResult], uploads: list[NodeIndexData] +) -> None: + assert uploads == build_node_index_datas( + hexdigests=set(hexdigests), snapshots=snapshot_results, node_indices=[0, 1, 2, 3] + ) @patch("astacus.common.utils.monotonic_time") -def test_backup_stats(mock_time, app, client): +def test_backup_stats(mock_time: Mock, app: FastAPI, client: TestClient) -> None: mock_time.side_effect = itertools.count(start=0.0, step=0.5) nodes = app.state.coordinator_config.nodes with respx.mock: diff --git a/tests/unit/coordinator/test_cleanup.py b/tests/unit/coordinator/test_cleanup.py index 2eda8b20..eec0fff4 100644 --- a/tests/unit/coordinator/test_cleanup.py +++ b/tests/unit/coordinator/test_cleanup.py @@ -6,6 +6,9 @@ """ from astacus.common import ipc +from astacus.common.rohmustorage import MultiRohmuStorage +from fastapi import FastAPI +from fastapi.testclient import TestClient import pytest import respx @@ -13,7 +16,16 @@ FAILS = [1, None] -def _run(*, client, populated_mstorage, app, fail_at=None, retention, exp_jsons, exp_digests): +def _run( + *, + client: TestClient, + populated_mstorage: MultiRohmuStorage, + app: FastAPI, + fail_at: int | None = None, + retention: ipc.Retention, + exp_jsons: int, + exp_digests: int, +) -> None: app.state.coordinator_config.retention = retention assert len(populated_mstorage.get_storage("x").list_jsons()) == 2 populated_mstorage.get_storage("x").upload_hexdigest_bytes("TOBEDELETED", b"x") @@ -44,7 +56,9 @@ def _run(*, client, populated_mstorage, app, fail_at=None, retention, exp_jsons, @pytest.mark.parametrize("fail_at", FAILS) -def test_api_cleanup_flow(fail_at, client, populated_mstorage, app): +def test_api_cleanup_flow( + fail_at: int | None, client: TestClient, populated_mstorage: MultiRohmuStorage, app: FastAPI +) -> None: _run( fail_at=fail_at, client=client, @@ -69,7 +83,9 @@ def test_api_cleanup_flow(fail_at, client, populated_mstorage, app): (ipc.Retention(maximum_backups=1, keep_days=10000), 1, 1), ], ) -def test_api_cleanup_retention(data, client, populated_mstorage, app): +def test_api_cleanup_retention( + data: tuple[ipc.Retention, int, int], client: TestClient, populated_mstorage: MultiRohmuStorage, app: FastAPI +) -> None: retention, exp_jsons, exp_digests = data _run( client=client, diff --git a/tests/unit/coordinator/test_list.py b/tests/unit/coordinator/test_list.py index 0f0e7f18..9c7cf166 100644 --- a/tests/unit/coordinator/test_list.py +++ b/tests/unit/coordinator/test_list.py @@ -23,13 +23,14 @@ from fastapi.testclient import TestClient from os import PathLike from pathlib import Path +from pytest_mock import MockerFixture from tests.utils import create_rohmu_config import datetime import pytest -def test_api_list(client: TestClient, populated_mstorage: MultiRohmuStorage, mocker) -> None: +def test_api_list(client: TestClient, populated_mstorage: MultiRohmuStorage, mocker: MockerFixture) -> None: assert populated_mstorage def _run(): diff --git a/tests/unit/coordinator/test_lock.py b/tests/unit/coordinator/test_lock.py index eebb89cc..a56fbd09 100644 --- a/tests/unit/coordinator/test_lock.py +++ b/tests/unit/coordinator/test_lock.py @@ -8,18 +8,20 @@ from astacus.common.magic import LockCall from astacus.common.statsd import StatsClient +from fastapi import FastAPI +from fastapi.testclient import TestClient from unittest.mock import patch import respx -def test_status_nonexistent(client): +def test_status_nonexistent(client: TestClient) -> None: response = client.get("/lock/123") assert response.status_code == 404 assert response.json() == {"detail": {"code": "operation_id_mismatch", "message": "Unknown operation id", "op": 123}} -def test_lock_no_nodes(app, client): +def test_lock_no_nodes(app: FastAPI, client: TestClient) -> None: nodes = app.state.coordinator_config.nodes nodes.clear() @@ -35,7 +37,7 @@ def test_lock_no_nodes(app, client): assert response.json() == {"state": "done"} -def test_lock_ok(app, client): +def test_lock_ok(app: FastAPI, client: TestClient) -> None: nodes = app.state.coordinator_config.nodes with respx.mock: for node in nodes: @@ -50,7 +52,7 @@ def test_lock_ok(app, client): assert app.state.coordinator_state.op_info.op_id == 1 -def test_lock_onefail(app, client): +def test_lock_onefail(app: FastAPI, client: TestClient) -> None: nodes = app.state.coordinator_config.nodes with respx.mock: for i, node in enumerate(nodes): diff --git a/tests/unit/coordinator/test_restore.py b/tests/unit/coordinator/test_restore.py index b6d4de02..c32c23c1 100644 --- a/tests/unit/coordinator/test_restore.py +++ b/tests/unit/coordinator/test_restore.py @@ -7,13 +7,16 @@ """ from astacus.common import exceptions, ipc from astacus.common.ipc import Plugin +from astacus.common.rohmustorage import MultiRohmuStorage from astacus.coordinator.config import CoordinatorNode from astacus.coordinator.plugins.base import get_node_to_backup_index from contextlib import nullcontext as does_not_raise from dataclasses import dataclass from datetime import datetime, UTC +from fastapi import FastAPI +from fastapi.testclient import TestClient from pathlib import Path -from typing import Callable, Optional +from typing import Any, Callable, ContextManager, Optional import httpx import json @@ -66,7 +69,7 @@ class RestoreTest: RestoreTest(partial=True), ], ) -def test_restore(rt, app, client, mstorage): +def test_restore(rt: RestoreTest, app: FastAPI, client: TestClient, mstorage: MultiRohmuStorage) -> None: # pylint: disable=too-many-statements # Create fake backup (not pretty but sufficient?) storage = mstorage.get_storage(rt.storage_name) @@ -124,7 +127,7 @@ def match_clear(request: httpx.Request) -> Optional[httpx.Response]: status_code=200 if rt.fail_at != 5 else 500, ) - req = {} + req: dict[str, Any] = {} if rt.storage_name: req["storage"] = rt.storage_name if rt.partial: @@ -173,7 +176,9 @@ def match_clear(request: httpx.Request) -> Optional[httpx.Response]: (["foo", "foo", "bar", "bar"], ["1", "3", "3", "3"], None, pytest.raises(exceptions.InsufficientNodesException)), ], ) -def test_node_to_backup_index(node_azlist, backup_azlist, expected_index, exception): +def test_node_to_backup_index( + node_azlist: list[str], backup_azlist: list[str], expected_index: list[int], exception: ContextManager +) -> None: snapshot_results = [ipc.SnapshotResult(az=az) for az in backup_azlist] nodes = [CoordinatorNode(url="unused", az=az) for az in node_azlist] with exception: @@ -204,7 +209,9 @@ def test_node_to_backup_index(node_azlist, backup_azlist, expected_index, except ({"backup_index": 1, "node_url": "url123"}, [None, None, 1], pytest.raises(exceptions.NotFoundException)), ], ) -def test_partial_node_to_backup_index(partial_node_spec, expected_index, exception): +def test_partial_node_to_backup_index( + partial_node_spec: dict[str, Any], expected_index: list[int | None], exception: ContextManager +) -> None: num_nodes = 3 snapshot_results = [ipc.SnapshotResult(hostname=f"host{i}") for i in range(num_nodes)] nodes = [CoordinatorNode(url=f"url{i}") for i in range(num_nodes)] diff --git a/tests/unit/node/conftest.py b/tests/unit/node/conftest.py index e1925334..d0682805 100644 --- a/tests/unit/node/conftest.py +++ b/tests/unit/node/conftest.py @@ -16,11 +16,12 @@ from fastapi.testclient import TestClient from pathlib import Path +import py import pytest @pytest.fixture(name="app") -def fixture_app(tmpdir) -> FastAPI: +def fixture_app(tmpdir: py.path.local) -> FastAPI: app = FastAPI() app.include_router(node_router, prefix="/node", tags=["node"]) root = Path(tmpdir) / "root" @@ -65,33 +66,33 @@ def fixture_uploader(storage): @pytest.fixture(name="storage") -def fixture_storage(tmpdir) -> FileStorage: +def fixture_storage(tmpdir: py.path.local) -> FileStorage: storage_path = Path(tmpdir) / "storage" storage_path.mkdir() return FileStorage(storage_path) @pytest.fixture(name="root") -def fixture_root(tmpdir) -> Path: +def fixture_root(tmpdir: py.path.local) -> Path: return Path(tmpdir) @pytest.fixture(name="src") -def fixture_src(tmpdir) -> Path: +def fixture_src(tmpdir: py.path.local) -> Path: src = Path(tmpdir) / "src" src.mkdir() return src @pytest.fixture(name="dst") -def fixture_dst(tmpdir) -> Path: +def fixture_dst(tmpdir: py.path.local) -> Path: dst = Path(tmpdir) / "dst" dst.mkdir() return dst @pytest.fixture(name="db") -def fixture_db(tmpdir) -> Path: +def fixture_db(tmpdir: py.path.local) -> Path: db = Path(tmpdir) / "db" return db diff --git a/tests/unit/node/test_node_cassandra.py b/tests/unit/node/test_node_cassandra.py index 74d0c4be..27910b89 100644 --- a/tests/unit/node/test_node_cassandra.py +++ b/tests/unit/node/test_node_cassandra.py @@ -8,40 +8,46 @@ from astacus.common.cassandra.utils import SYSTEM_KEYSPACES from astacus.node.api import READONLY_SUBOPS from astacus.node.config import CassandraAccessLevel, CassandraNodeConfig +from fastapi import FastAPI +from fastapi.testclient import TestClient +from pytest_mock import MockerFixture +from requests import Response from tests.unit.conftest import CassandraTestConfig -from typing import Sequence +from types import ModuleType +from typing import Callable, Sequence +import py import pytest import subprocess @pytest.fixture(name="astacus_node_cassandra", autouse=True) -def fixture_astacus_node_cassandra() -> None: +def fixture_astacus_node_cassandra() -> ModuleType: return pytest.importorskip("astacus.node.cassandra") class CassandraTestEnv(CassandraTestConfig): cassandra_node_config: CassandraNodeConfig - def __init__(self, *, app, client, mocker, tmpdir): + def __init__(self, *, app: FastAPI, client: TestClient, mocker: MockerFixture, tmpdir: py.path.local) -> None: super().__init__(mocker=mocker, tmpdir=tmpdir) self.app = app self.client = client - def lock(self): + def lock(self) -> None: response = self.client.post("/node/lock?locker=x&ttl=10") assert response.status_code == 200, response.json() - def post(self, *, subop, **kwargs): + def post(self, *, subop: str, **kwargs) -> Response: url = f"/node/cassandra/{subop}" return self.client.post(url, **kwargs) - def get_status(self, response): + def get_status(self, response: Response) -> Response: assert response.status_code == 200, response.json() status_url = response.json()["status_url"] return self.client.get(status_url) - def setup_cassandra_node_config(self): + def setup_cassandra_node_config(self) -> None: self.cassandra_node_config = CassandraNodeConfig( client=self.cassandra_client_config, nodetool_command=["nodetool"], @@ -53,14 +59,16 @@ def setup_cassandra_node_config(self): @pytest.fixture(name="ctenv") -def fixture_ctenv(app, client, mocker, tmpdir): +def fixture_ctenv(app: FastAPI, client: TestClient, mocker: MockerFixture, tmpdir: py.path.local) -> CassandraTestEnv: return CassandraTestEnv(app=app, client=client, mocker=mocker, tmpdir=tmpdir) @pytest.mark.parametrize( "subop", set(ipc.CassandraSubOp) - {ipc.CassandraSubOp.get_schema_hash, ipc.CassandraSubOp.restore_sstables} ) -def test_api_cassandra_subop(app, ctenv, mocker, subop): +def test_api_cassandra_subop( + app: FastAPI, ctenv: CassandraTestEnv, mocker: MockerFixture, subop: ipc.CassandraSubOp +) -> None: req_json = {"tokens": ["42", "7"]} # Without lock, we shouldn't be able to do use the endpoint @@ -124,7 +132,9 @@ def test_api_cassandra_subop(app, ctenv, mocker, subop): @pytest.mark.parametrize("fail", [True]) -def test_api_cassandra_get_schema_hash(ctenv, fail, mocker, astacus_node_cassandra): +def test_api_cassandra_get_schema_hash( + ctenv: CassandraTestEnv, fail: bool, mocker: MockerFixture, astacus_node_cassandra: ModuleType +) -> None: # The state of API *before* these two setup steps are done is checked in the test_api_cassandra_subop ctenv.lock() ctenv.setup_cassandra_node_config() @@ -149,7 +159,9 @@ def test_api_cassandra_get_schema_hash(ctenv, fail, mocker, astacus_node_cassand class TestCassandraRestoreSSTables: @pytest.fixture(name="make_sstables_request") - def fixture_make_sstables_request(self, astacus_node_cassandra) -> type[ipc.CassandraRestoreSSTablesRequest]: + def fixture_make_sstables_request( + self, astacus_node_cassandra: ModuleType + ) -> Callable[..., ipc.CassandraRestoreSSTablesRequest]: class DefaultedRestoreSSTablesRequest(ipc.CassandraRestoreSSTablesRequest): table_glob: str = astacus_node_cassandra.SNAPSHOT_GLOB keyspaces_to_skip: Sequence[str] = list(SYSTEM_KEYSPACES) @@ -159,38 +171,46 @@ class DefaultedRestoreSSTablesRequest(ipc.CassandraRestoreSSTablesRequest): return DefaultedRestoreSSTablesRequest @pytest.fixture(name="locked_ctenv", autouse=True) - def fixture_locked_ctenv(self, ctenv) -> None: + def fixture_locked_ctenv(self, ctenv: CassandraTestEnv) -> None: ctenv.lock() ctenv.setup_cassandra_node_config() - def test_uses_glob_from_request_instead_of_default(self, ctenv, make_sstables_request) -> None: + def test_uses_glob_from_request_instead_of_default( + self, ctenv: CassandraTestEnv, make_sstables_request: Callable[..., ipc.CassandraRestoreSSTablesRequest] + ) -> None: req = make_sstables_request(table_glob=f"data/*/dummytable-123/snapshots/{SNAPSHOT_NAME}") self.assert_request_succeeded(ctenv, req) assert (ctenv.root / "data" / "dummyks" / "dummytable-234" / "asdf").read_text() == "foobar" assert not (ctenv.root / "data" / "dummyks" / "anothertable-789" / "data.file").exists() - def test_skips_keyspace_if_told_to(self, ctenv, make_sstables_request) -> None: + def test_skips_keyspace_if_told_to( + self, ctenv: CassandraTestEnv, make_sstables_request: Callable[..., ipc.CassandraRestoreSSTablesRequest] + ) -> None: req = make_sstables_request(keyspaces_to_skip=["dummyks"]) self.assert_request_succeeded(ctenv, req) assert not (ctenv.root / "data" / "dummyks" / "dummytable-234" / "asdf").exists() assert (ctenv.root / "data" / "system_schema" / "tables-789" / "data.file").read_text() == "schema" - def test_matches_tables_by_id_when_told_to(self, ctenv, make_sstables_request) -> None: + def test_matches_tables_by_id_when_told_to( + self, ctenv: CassandraTestEnv, make_sstables_request: Callable[..., ipc.CassandraRestoreSSTablesRequest] + ) -> None: req = make_sstables_request(match_tables_by=ipc.CassandraTableMatching.cfid) self.assert_request_succeeded(ctenv, req) assert (ctenv.root / "data" / "dummyks" / "dummytable-123" / "asdf").read_text() == "foobar" assert not (ctenv.root / "data" / "dummyks" / "dummytable-234" / "asdf").exists() - def test_allows_existing_files_when_told_to(self, ctenv, make_sstables_request) -> None: + def test_allows_existing_files_when_told_to( + self, ctenv: CassandraTestEnv, make_sstables_request: Callable[..., ipc.CassandraRestoreSSTablesRequest] + ) -> None: req = make_sstables_request(expect_empty_target=False) (ctenv.root / "data" / "dummyks" / "dummytable-123" / "existing_file").write_text("exists") self.assert_request_succeeded(ctenv, req) - def assert_request_succeeded(self, ctenv, req: ipc.CassandraRestoreSSTablesRequest) -> None: + def assert_request_succeeded(self, ctenv: CassandraTestEnv, req: ipc.CassandraRestoreSSTablesRequest) -> None: response = ctenv.post(subop=ipc.CassandraSubOp.restore_sstables, json=req.dict()) status = ctenv.get_status(response) assert status.status_code == 200, response.json() @@ -201,7 +221,7 @@ def assert_request_succeeded(self, ctenv, req: ipc.CassandraRestoreSSTablesReque @pytest.mark.parametrize("dangerous_op", set(ipc.CassandraSubOp) - READONLY_SUBOPS) -def test_dangerous_ops_not_allowed_on_read_access_level(ctenv, dangerous_op: ipc.CassandraSubOp): +def test_dangerous_ops_not_allowed_on_read_access_level(ctenv: CassandraTestEnv, dangerous_op: ipc.CassandraSubOp) -> None: ctenv.lock() ctenv.setup_cassandra_node_config() ctenv.cassandra_node_config.access_level = CassandraAccessLevel.read diff --git a/tests/unit/node/test_node_lock.py b/tests/unit/node/test_node_lock.py index e90302a8..15dd9639 100644 --- a/tests/unit/node/test_node_lock.py +++ b/tests/unit/node/test_node_lock.py @@ -4,7 +4,10 @@ """ -def test_api_lock_unlock(client): +from fastapi.testclient import TestClient + + +def test_api_lock_unlock(client: TestClient) -> None: # Play with lock response = client.post("/node/lock") assert response.status_code == 422, response.json() diff --git a/tests/unit/node/test_node_snapshot.py b/tests/unit/node/test_node_snapshot.py index 683879b3..bcc3e15b 100644 --- a/tests/unit/node/test_node_snapshot.py +++ b/tests/unit/node/test_node_snapshot.py @@ -27,7 +27,7 @@ def test_snapshot( dst: Path, db: Path, src_is_dst: bool, -): +) -> None: if src_is_dst: dst = src @@ -89,7 +89,7 @@ def test_snapshot( assert not hashes_empty -def test_api_snapshot_and_upload(client: TestClient, mocker: MockerFixture): +def test_api_snapshot_and_upload(client: TestClient, mocker: MockerFixture) -> None: url = "http://addr/result" m = mocker.patch.object(utils, "http_request") response = client.post("/node/snapshot") @@ -129,7 +129,7 @@ def test_api_snapshot_and_upload(client: TestClient, mocker: MockerFixture): assert result2.progress.finished_successfully -def test_api_snapshot_error(client, mocker): +def test_api_snapshot_error(client: TestClient, mocker: MockerFixture) -> None: req_json = {"root_globs": ["*"]} response = client.post("/node/lock?locker=x&ttl=10") assert response.status_code == 200, response.json() @@ -170,7 +170,7 @@ def test_snapshot_file_size_changed( src_is_dst: bool, truncate_to, hashes_in_second_snapshot: int, -): +) -> None: if src_is_dst: dst = src diff --git a/tests/utils.py b/tests/utils.py index e79f59e7..551a2e65 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,17 +4,18 @@ """ from astacus.common.rohmustorage import RohmuConfig from pathlib import Path -from typing import List, Union +from typing import Final, List, Union import importlib import os +import py import re import subprocess import sys # These test keys are from copied from pghoard -CONSTANT_TEST_RSA_PUBLIC_KEY = """\ +CONSTANT_TEST_RSA_PUBLIC_KEY: Final = """\ -----BEGIN PUBLIC KEY----- MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDQ9yu7rNmu0GFMYeQq9Jo2B3d9 hv5t4a+54TbbxpJlks8T27ipgsaIjqiQP7+uXNfU6UCzGFEHs9R5OELtO3Hq0Dn+ @@ -22,7 +23,7 @@ lWN+9KPe+5bXS8of+wIDAQAB -----END PUBLIC KEY-----""" -CONSTANT_TEST_RSA_PRIVATE_KEY = """\ +CONSTANT_TEST_RSA_PRIVATE_KEY: Final = """\ -----BEGIN PRIVATE KEY----- MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAND3K7us2a7QYUxh 5Cr0mjYHd32G/m3hr7nhNtvGkmWSzxPbuKmCxoiOqJA/v65c19TpQLMYUQez1Hk4 @@ -41,7 +42,7 @@ -----END PRIVATE KEY-----""" -def create_rohmu_config(tmpdir, *, compression=True, encryption=True) -> RohmuConfig: +def create_rohmu_config(tmpdir: py.path.local, *, compression: bool = True, encryption: bool = True) -> RohmuConfig: x_path = Path(tmpdir) / "rohmu-x" x_path.mkdir(exist_ok=True) y_path = Path(tmpdir) / "rohmu-y"