diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 27ebcefd..7423a11b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -46,16 +46,21 @@ jobs: run: cd /tmp/DIRACRepo && ./integration_tests.py install-server - name: Install client run: cd /tmp/DIRACRepo && ./integration_tests.py install-client + - name: Install pilot + run: cd /tmp/DIRACRepo && ./integration_tests.py install-pilot - name: Server tests run: cd /tmp/DIRACRepo && ./integration_tests.py test-server || touch server-tests-failed - name: Client tests run: cd /tmp/DIRACRepo && ./integration_tests.py test-client || touch client-tests-failed + - name: Pilot tests + run: cd /tmp/DIRACRepo && ./integration_tests.py test-pilot || touch pilot-tests-failed - name: Check test status run: | has_error=0 # TODO: set has_error=1 when we are ready to really run the tests if [ -f server-tests-failed ]; then has_error=0; echo "Server tests failed"; fi if [ -f client-tests-failed ]; then has_error=0; echo "Client tests failed"; fi + if [ -f pilot-tests-failed ]; then has_error=0; echo "Pilot tests failed"; fi if [ ${has_error} = 1 ]; then exit 1; fi - name: diracx logs if: ${{ failure() }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 40c82ce6..494b1136 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,6 +14,21 @@ defaults: shell: bash -el {0} jobs: + + shellcheck: + runs-on: ubuntu-latest + if: github.event_name != 'push' || github.repository == 'DIRACGrid/diracx' + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v4 + - name: Run shellcheck + # Excluded codes related to sourcing files + # SC1090: Can't follow non-constant source + # SC1091: Not following sourced file + run: | + find -name '*.sh' -print0 | xargs -0 -n1 shellcheck --exclude=SC1090,SC1091 --external-source + pytest: name: Unit test - ${{ matrix.package }} runs-on: ubuntu-latest @@ -39,13 +54,7 @@ jobs: - uses: mamba-org/setup-micromamba@v1 with: # TODO: Use a conda environment file used for the diracx/base container image - environment-name: test-env - create-args: >- - python=3.11 - m2crypto - python-gfal2 - mypy - pip + environment-file: environment.yml init-shell: bash post-cleanup: 'all' - name: Set up environment diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 50c3b81b..24ff74a7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks default_language_version: - python: python3 + python: python3.11 repos: - repo: https://github.com/pre-commit/pre-commit-hooks @@ -13,18 +13,18 @@ repos: - id: check-added-large-files - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.5.0' + rev: v0.6.2 hooks: - id: ruff args: ["--fix"] - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.10.1 + rev: v1.11.1 hooks: - id: mypy additional_dependencies: @@ -36,3 +36,10 @@ repos: - types-aiobotocore[essential] - boto3-stubs[essential] exclude: ^(diracx-client/src/diracx/client/|diracx-[a-z]+/tests/|diracx-testing/|build) + + - repo: https://github.com/asottile/pyupgrade + rev: v3.17.0 + hooks: + - id: pyupgrade + args: ["--py311-plus"] + exclude: ^(diracx-client/src/diracx/client/|diracx-cli/tests/|diracx-cli/src/diracx/cli/) diff --git a/diracx-api/pyproject.toml b/diracx-api/pyproject.toml index a3e8960a..ef36c330 100644 --- a/diracx-api/pyproject.toml +++ b/diracx-api/pyproject.toml @@ -2,7 +2,7 @@ name = "diracx-api" description = "TODO" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" keywords = [] license = {text = "GPL-3.0-only"} classifiers = [ diff --git a/diracx-cli/pyproject.toml b/diracx-cli/pyproject.toml index 431d0a01..e6806039 100644 --- a/diracx-cli/pyproject.toml +++ b/diracx-cli/pyproject.toml @@ -2,7 +2,7 @@ name = "diracx-cli" description = "TODO" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" keywords = [] license = {text = "GPL-3.0-only"} classifiers = [ @@ -20,7 +20,7 @@ dependencies = [ "gitpython", "pydantic", "rich", - "typer", + "typer <0.12.4", "pyyaml", ] dynamic = ["version"] diff --git a/diracx-client/pyproject.toml b/diracx-client/pyproject.toml index 02cf6c3a..1763228d 100644 --- a/diracx-client/pyproject.toml +++ b/diracx-client/pyproject.toml @@ -2,7 +2,7 @@ name = "diracx-client" description = "TODO" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" keywords = [] license = {text = "GPL-3.0-only"} classifiers = [ diff --git a/diracx-core/pyproject.toml b/diracx-core/pyproject.toml index 846d96af..26e1e11d 100644 --- a/diracx-core/pyproject.toml +++ b/diracx-core/pyproject.toml @@ -2,7 +2,7 @@ name = "diracx-core" description = "Common code used by all DiracX packages" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" keywords = [] license = {text = "GPL-3.0-only"} classifiers = [ diff --git a/diracx-core/src/diracx/core/config/schema.py b/diracx-core/src/diracx/core/config/schema.py index 29c1c514..84c24720 100644 --- a/diracx-core/src/diracx/core/config/schema.py +++ b/diracx-core/src/diracx/core/config/schema.py @@ -2,7 +2,7 @@ import os from datetime import datetime -from typing import Annotated, Any, Optional, TypeVar +from typing import Annotated, Any, TypeVar from pydantic import BaseModel as _BaseModel from pydantic import ConfigDict, EmailStr, Field, PrivateAttr, model_validator @@ -65,10 +65,10 @@ class GroupConfig(BaseModel): AutoUploadProxy: bool = False JobShare: int = 1000 Properties: SerializableSet[SecurityProperty] - Quota: Optional[int] = None + Quota: int | None = None Users: SerializableSet[str] AllowBackgroundTQs: bool = False - VOMSRole: Optional[str] = None + VOMSRole: str | None = None AutoSyncVOMS: bool = False diff --git a/diracx-core/src/diracx/core/properties.py b/diracx-core/src/diracx/core/properties.py index 3a80b879..c75c014e 100644 --- a/diracx-core/src/diracx/core/properties.py +++ b/diracx-core/src/diracx/core/properties.py @@ -6,7 +6,8 @@ import inspect import operator -from typing import Any, Callable +from collections.abc import Callable +from typing import Any from pydantic import GetCoreSchemaHandler from pydantic_core import CoreSchema, core_schema diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index 936dad05..3ee37df8 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -7,8 +7,9 @@ ) import contextlib +from collections.abc import AsyncIterator from pathlib import Path -from typing import Annotated, Any, AsyncIterator, Self, TypeVar +from typing import Annotated, Any, Self, TypeVar from authlib.jose import JsonWebKey from cryptography.fernet import Fernet diff --git a/diracx-core/tests/test_secrets.py b/diracx-core/tests/test_secrets.py index 9b2f0c50..59724c8b 100644 --- a/diracx-core/tests/test_secrets.py +++ b/diracx-core/tests/test_secrets.py @@ -15,8 +15,7 @@ def compare_keys(key1, key2): def test_token_signing_key(tmp_path): private_key = rsa.generate_private_key( public_exponent=65537, - # DANGER: 512-bits is a bad idea for prod but makes the test notably faster! - key_size=512, + key_size=1024, ) private_key_pem = private_key.private_bytes( encoding=serialization.Encoding.PEM, diff --git a/diracx-db/pyproject.toml b/diracx-db/pyproject.toml index f7a145fc..37d728be 100644 --- a/diracx-db/pyproject.toml +++ b/diracx-db/pyproject.toml @@ -2,7 +2,7 @@ name = "diracx-db" description = "TODO" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" keywords = [] license = {text = "GPL-3.0-only"} classifiers = [ diff --git a/diracx-db/src/diracx/db/os/utils.py b/diracx-db/src/diracx/db/os/utils.py index 1033a944..659c20bd 100644 --- a/diracx-db/src/diracx/db/os/utils.py +++ b/diracx-db/src/diracx/db/os/utils.py @@ -7,9 +7,10 @@ import logging import os from abc import ABCMeta, abstractmethod +from collections.abc import AsyncIterator from contextvars import ContextVar from datetime import datetime -from typing import Any, AsyncIterator, Self +from typing import Any, Self from opensearchpy import AsyncOpenSearch diff --git a/diracx-db/src/diracx/db/sql/jobs/db.py b/diracx-db/src/diracx/db/sql/jobs/db.py index 38938ddb..36ba6cc1 100644 --- a/diracx-db/src/diracx/db/sql/jobs/db.py +++ b/diracx-db/src/diracx/db/sql/jobs/db.py @@ -503,9 +503,7 @@ async def set_properties( """ # Check that all we always update the same set of properties - required_parameters_set = set( - [tuple(sorted(k.keys())) for k in properties.values()] - ) + required_parameters_set = {tuple(sorted(k.keys())) for k in properties.values()} if len(required_parameters_set) != 1: raise NotImplementedError( @@ -682,10 +680,10 @@ async def get_tq_infos_for_jobs( .join(JobsQueue, TaskQueues.TQId == JobsQueue.TQId) .where(JobsQueue.JobId.in_(job_ids)) ) - return set( + return { (int(row[0]), str(row[1]), str(row[2]), str(row[3])) for row in (await self.conn.execute(stmt)).all() - ) + } async def get_owner_for_task_queue(self, tq_id: int) -> dict[str, str]: """Get the owner and owner group for a task queue.""" diff --git a/diracx-db/src/diracx/db/sql/utils.py b/diracx-db/src/diracx/db/sql/utils.py index 9dba7b8a..6c16e385 100644 --- a/diracx-db/src/diracx/db/sql/utils.py +++ b/diracx-db/src/diracx/db/sql/utils.py @@ -6,10 +6,11 @@ import logging import os from abc import ABCMeta +from collections.abc import AsyncIterator from contextvars import ContextVar from datetime import datetime, timedelta, timezone from functools import partial -from typing import TYPE_CHECKING, AsyncIterator, Self, cast +from typing import TYPE_CHECKING, Self, cast from pydantic import TypeAdapter from sqlalchemy import Column as RawColumn diff --git a/diracx-routers/pyproject.toml b/diracx-routers/pyproject.toml index 97fdccb3..6418dde1 100644 --- a/diracx-routers/pyproject.toml +++ b/diracx-routers/pyproject.toml @@ -2,7 +2,7 @@ name = "diracx-routers" description = "TODO" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" keywords = [] license = { text = "GPL-3.0-only" } classifiers = [ diff --git a/diracx-routers/src/diracx/routers/__init__.py b/diracx-routers/src/diracx/routers/__init__.py index 2ff15cde..26a83263 100644 --- a/diracx-routers/src/diracx/routers/__init__.py +++ b/diracx-routers/src/diracx/routers/__init__.py @@ -10,10 +10,10 @@ import inspect import logging import os -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, Awaitable, Callable, Iterable, Sequence from functools import partial from logging import Formatter, StreamHandler -from typing import Any, Awaitable, Callable, Iterable, Sequence, TypeVar, cast +from typing import Any, TypeVar, cast import dotenv from cachetools import TTLCache @@ -346,12 +346,10 @@ def create_app() -> DiracFastAPI: # Find all the access policies - available_access_policy_names = set( - [ - entry_point.name - for entry_point in select_from_extension(group="diracx.access_policies") - ] - ) + available_access_policy_names = { + entry_point.name + for entry_point in select_from_extension(group="diracx.access_policies") + } all_access_policies = {} @@ -428,7 +426,7 @@ async def is_db_unavailable(db: BaseSQLDB | BaseOSDB) -> str: return _db_alive_cache[db] -async def db_transaction(db: T2) -> AsyncGenerator[T2, None]: +async def db_transaction(db: T2) -> AsyncGenerator[T2]: """Initiate a DB transaction.""" # Entering the context already triggers a connection to the DB # that may fail diff --git a/diracx-routers/src/diracx/routers/access_policies.py b/diracx-routers/src/diracx/routers/access_policies.py index cb4d5a7f..b7605fe7 100644 --- a/diracx-routers/src/diracx/routers/access_policies.py +++ b/diracx-routers/src/diracx/routers/access_policies.py @@ -21,7 +21,8 @@ import os import time from abc import ABCMeta, abstractmethod -from typing import Annotated, Callable, Self +from collections.abc import Callable +from typing import Annotated, Self from fastapi import Depends diff --git a/diracx-routers/src/diracx/routers/job_manager/access_policies.py b/diracx-routers/src/diracx/routers/job_manager/access_policies.py index 40d6a75b..68ff4a91 100644 --- a/diracx-routers/src/diracx/routers/job_manager/access_policies.py +++ b/diracx-routers/src/diracx/routers/job_manager/access_policies.py @@ -1,7 +1,8 @@ from __future__ import annotations +from collections.abc import Callable from enum import StrEnum, auto -from typing import Annotated, Callable +from typing import Annotated from fastapi import Depends, HTTPException, status diff --git a/diracx-routers/src/diracx/routers/job_manager/sandboxes.py b/diracx-routers/src/diracx/routers/job_manager/sandboxes.py index 6e62379c..43f3545e 100644 --- a/diracx-routers/src/diracx/routers/job_manager/sandboxes.py +++ b/diracx-routers/src/diracx/routers/job_manager/sandboxes.py @@ -1,8 +1,9 @@ from __future__ import annotations import contextlib +from collections.abc import AsyncIterator from http import HTTPStatus -from typing import TYPE_CHECKING, Annotated, AsyncIterator, Literal +from typing import TYPE_CHECKING, Annotated, Literal from aiobotocore.session import get_session from botocore.config import Config diff --git a/diracx-routers/tests/auth/test_standard.py b/diracx-routers/tests/auth/test_standard.py index 75faf11d..431437b3 100644 --- a/diracx-routers/tests/auth/test_standard.py +++ b/diracx-routers/tests/auth/test_standard.py @@ -527,8 +527,7 @@ async def test_refresh_token_invalid(test_client, auth_httpx_mock: HTTPXMock): # Encode it differently (using another algorithm) private_key = rsa.generate_private_key( public_exponent=65537, - # DANGER: 512-bits is a bad idea for prod but makes the test notably faster! - key_size=512, + key_size=1024, ) pem = private_key.private_bytes( encoding=serialization.Encoding.PEM, diff --git a/diracx-routers/tests/test_job_manager.py b/diracx-routers/tests/test_job_manager.py index 5a32a64f..fac49ba6 100644 --- a/diracx-routers/tests/test_job_manager.py +++ b/diracx-routers/tests/test_job_manager.py @@ -569,7 +569,7 @@ def test_set_job_status_offset_naive_datetime_return_bad_request( valid_job_id: int, ): # Act - date = datetime.utcnow().isoformat(sep=" ") # noqa: DTZ003 + date = datetime.now(tz=timezone.utc).isoformat(sep=" ").split("+")[0] r = normal_user_client.patch( f"/api/jobs/{valid_job_id}/status", json={ diff --git a/diracx-testing/pyproject.toml b/diracx-testing/pyproject.toml index af5fa28d..0c841fcf 100644 --- a/diracx-testing/pyproject.toml +++ b/diracx-testing/pyproject.toml @@ -2,7 +2,7 @@ name = "diracx-testing" description = "TODO" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" keywords = [] license = {text = "GPL-3.0-only"} classifiers = [ diff --git a/environment.yml b/environment.yml index b80239ca..52363911 100644 --- a/environment.yml +++ b/environment.yml @@ -4,6 +4,7 @@ channels: - conda-forge - nodefaults dependencies: + - python =3.11 - authlib - aiohttp - aiomysql @@ -31,7 +32,13 @@ dependencies: - isodate - mypy - opensearch-py + - opentelemetry-api + - opentelemetry-exporter-otlp + - opentelemetry-instrumentation-fastapi + - opentelemetry-instrumentation-logging + - pre-commit - pydantic >=2.4 + - pydantic-settings - pyjwt - pytest - pytest-asyncio @@ -45,7 +52,9 @@ dependencies: - requests - rich - sqlalchemy - - typer + - shellcheck + # Exclude version from 0.12.4 because of https://github.com/DIRACGrid/diracx/issues/280 + - typer <0.12.4 - types-cachetools - types-PyYAML - types-requests diff --git a/pyproject.toml b/pyproject.toml index 92ac232b..ab8f291c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "diracx" description = "Client installation for users of DiracX installations" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" keywords = [] license = { text = "GPL-3.0-only" } classifiers = [ diff --git a/run_local.sh b/run_local.sh index 30d81225..fa4ef69f 100755 --- a/run_local.sh +++ b/run_local.sh @@ -34,7 +34,8 @@ export DIRACX_DB_URL_SANDBOXMETADATADB="sqlite+aiosqlite:///:memory:" export DIRACX_DB_URL_TASKQUEUEDB="sqlite+aiosqlite:///:memory:" export DIRACX_SERVICE_AUTH_TOKEN_KEY="file://${signing_key}" export DIRACX_SERVICE_AUTH_STATE_KEY="${state_key}" -export DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS='["http://'$(hostname| tr -s '[:upper:]' '[:lower:]')':8000/docs/oauth2-redirect"]' +hostname_lower=$(hostname | tr -s '[:upper:]' '[:lower:]') +export DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS='["http://'"$hostname_lower"':8000/docs/oauth2-redirect"]' export DIRACX_SANDBOX_STORE_BUCKET_NAME=sandboxes export DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET=true export DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS='{"endpoint_url": "http://localhost:3000", "aws_access_key_id": "console", "aws_secret_access_key": "console123"}' @@ -45,7 +46,7 @@ uvicorn --factory diracx.routers:create_app --reload & diracx_pid=$! success=0 -for i in {1..10}; do +for _ in {1..10}; do if curl --silent --head http://localhost:8000 > /dev/null; then success=1 break