diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2483ff311..23c3381d8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -245,3 +245,20 @@ jobs: environments: ${{ matrix.package }}-generate-client - name: Run autorest run: pixi run -e ${{ matrix.package }}-generate-client generate-client + + pre-commit: + name: pre-commit (with pixi) + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4 + with: + run-install: false + post-cleanup: false + - uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4 + with: + cache: false + environments: pre-commit + - name: Run pre-commit + run: pixi run pre-commit run --all-files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1fd11f86d..dd4865dde 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: python: python3 ci: - skip: [generate-pixi-docs, settings-doc-check] + skip: [generate-pixi-docs, settings-doc-check, check-init-files] default_stages: [pre-commit] @@ -90,7 +90,7 @@ repos: hooks: - id: generate-pixi-docs name: Generate pixi tasks documentation - entry: pixi run -e default python .github/workflows/generate_pixi_tasks_doc.py + entry: pixi run -e default python scripts/generate_pixi_tasks_doc.py language: system pass_filenames: false files: ^pixi\.toml$|^pixi\.lock$ # only run if pixi files change @@ -103,3 +103,12 @@ repos: language: system pass_filenames: false files: ^(diracx-.*/src/diracx/.*/settings\.py|diracx-routers/src/diracx/routers/otel\.py|docs/.*\.j2|docs/templates/.*\.jinja|scripts/generate_settings_docs\.py)$ + + - repo: local + hooks: + - id: check-init-files + name: Check __all__ dunder in __init__.py files + language: system + entry: pixi run -e default python scripts/check_init_files_precommit_hook.py + files: ^(diracx-|extensions/gubbins/gubbins-)[a-z]+/src/[a-z]+/[a-z]+/((.*/)*__init__\.py|[a-zA-Z0-9_]+\.py)$ + exclude: (test|_generated|__main__) diff --git a/diracx-api/src/diracx/api/__init__.py b/diracx-api/src/diracx/api/__init__.py index 329dd5739..4d21ee850 100644 --- a/diracx-api/src/diracx/api/__init__.py +++ b/diracx-api/src/diracx/api/__init__.py @@ -1,5 +1,3 @@ from __future__ import annotations -__all__ = ("jobs",) - -from . import jobs +__all__ = [] diff --git a/diracx-api/src/diracx/api/jobs.py b/diracx-api/src/diracx/api/jobs.py index 181b7b9b7..677671360 100644 --- a/diracx-api/src/diracx/api/jobs.py +++ b/diracx-api/src/diracx/api/jobs.py @@ -1,6 +1,6 @@ from __future__ import annotations -__all__ = ("create_sandbox", "download_sandbox") +__all__ = ["create_sandbox", "download_sandbox"] import hashlib import logging diff --git a/diracx-api/src/diracx/api/utils.py b/diracx-api/src/diracx/api/utils.py index 8e70be70e..3e78d3cd7 100644 --- a/diracx-api/src/diracx/api/utils.py +++ b/diracx-api/src/diracx/api/utils.py @@ -1,6 +1,6 @@ from __future__ import annotations -__all__ = ("with_client",) +__all__ = ["with_client"] from functools import wraps diff --git a/diracx-cli/src/diracx/cli/__init__.py b/diracx-cli/src/diracx/cli/__init__.py index 632c1ff13..fb84fac39 100644 --- a/diracx-cli/src/diracx/cli/__init__.py +++ b/diracx-cli/src/diracx/cli/__init__.py @@ -1,12 +1,11 @@ from __future__ import annotations +__all__ = ["app"] + from diracx.core.extensions import DiracEntryPoint, select_from_extension from .auth import app -__all__ = ("app",) - - # Load all the sub commands cli_names = set( [ diff --git a/diracx-cli/src/diracx/cli/auth.py b/diracx-cli/src/diracx/cli/auth.py index 1ed22586b..74323d38e 100644 --- a/diracx-cli/src/diracx/cli/auth.py +++ b/diracx-cli/src/diracx/cli/auth.py @@ -1,6 +1,6 @@ from __future__ import annotations -__all__ = ("app",) +__all__ = ["app"] import asyncio import json diff --git a/diracx-cli/src/diracx/cli/config.py b/diracx-cli/src/diracx/cli/config.py index 9fa2cda02..fbe3c6507 100644 --- a/diracx-cli/src/diracx/cli/config.py +++ b/diracx-cli/src/diracx/cli/config.py @@ -2,7 +2,7 @@ # from __future__ import annotations from __future__ import annotations -__all__ = ("dump",) +__all__ = ["dump"] import json diff --git a/diracx-cli/src/diracx/cli/internal/__init__.py b/diracx-cli/src/diracx/cli/internal/__init__.py index d89a2e44c..67640ddd5 100644 --- a/diracx-cli/src/diracx/cli/internal/__init__.py +++ b/diracx-cli/src/diracx/cli/internal/__init__.py @@ -1,8 +1,8 @@ from __future__ import annotations +__all__ = ["app"] + from . import legacy from .config import app -__all__ = ("app",) - app.add_typer(legacy.app, name="legacy") diff --git a/diracx-cli/src/diracx/cli/internal/config.py b/diracx-cli/src/diracx/cli/internal/config.py index e454e88e7..96f68ad3a 100644 --- a/diracx-cli/src/diracx/cli/internal/config.py +++ b/diracx-cli/src/diracx/cli/internal/config.py @@ -8,9 +8,10 @@ import yaml from pydantic import TypeAdapter -from diracx.core.config import ConfigSource, ConfigSourceUrl -from diracx.core.config.schema import ( +from diracx.core.config import ( Config, + ConfigSource, + ConfigSourceUrl, DIRACConfig, GroupConfig, IdpConfig, diff --git a/diracx-cli/src/diracx/cli/internal/legacy.py b/diracx-cli/src/diracx/cli/internal/legacy.py index 6d1bef51b..a12bf023a 100644 --- a/diracx-cli/src/diracx/cli/internal/legacy.py +++ b/diracx-cli/src/diracx/cli/internal/legacy.py @@ -16,11 +16,10 @@ if TYPE_CHECKING: from diraccfg.cfg import CFGAsDict -from pydantic import BaseModel +from pydantic import BaseModel, Field from typer import Option -from diracx.core.config import Config -from diracx.core.config.schema import Field, SupportInfo +from diracx.core.config import Config, SupportInfo from diracx.core.extensions import DiracEntryPoint, select_from_extension from ..utils import AsyncTyper diff --git a/diracx-cli/src/diracx/cli/jobs.py b/diracx-cli/src/diracx/cli/jobs.py index 1102bd7e8..862f7b908 100644 --- a/diracx-cli/src/diracx/cli/jobs.py +++ b/diracx-cli/src/diracx/cli/jobs.py @@ -2,7 +2,7 @@ # from __future__ import annotations from __future__ import annotations -__all__ = ("app",) +__all__ = ["app"] import json import re @@ -13,7 +13,7 @@ from typer import FileText, Option from diracx.client.aio import AsyncDiracClient -from diracx.core.models.search import ( +from diracx.core.models import ( ScalarSearchOperator, SearchSpec, VectorSearchOperator, diff --git a/diracx-cli/src/diracx/cli/utils.py b/diracx-cli/src/diracx/cli/utils.py index afc59692b..a21908a91 100644 --- a/diracx-cli/src/diracx/cli/utils.py +++ b/diracx-cli/src/diracx/cli/utils.py @@ -1,6 +1,6 @@ from __future__ import annotations -__all__ = ("AsyncTyper",) +__all__ = ["AsyncTyper"] from asyncio import run from functools import wraps diff --git a/diracx-cli/tests/legacy/__init__.py b/diracx-cli/tests/legacy/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-cli/tests/legacy/__init__.py +++ b/diracx-cli/tests/legacy/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-cli/tests/legacy/cs_sync/__init__.py b/diracx-cli/tests/legacy/cs_sync/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-cli/tests/legacy/cs_sync/__init__.py +++ b/diracx-cli/tests/legacy/cs_sync/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-cli/tests/legacy/cs_sync/test_cssync.py b/diracx-cli/tests/legacy/cs_sync/test_cssync.py index fb06de7df..e89c1a480 100644 --- a/diracx-cli/tests/legacy/cs_sync/test_cssync.py +++ b/diracx-cli/tests/legacy/cs_sync/test_cssync.py @@ -6,7 +6,7 @@ from typer.testing import CliRunner from diracx.cli import app -from diracx.core.config.schema import Config +from diracx.core.config import Config runner = CliRunner() diff --git a/diracx-cli/tests/test_login.py b/diracx-cli/tests/test_login.py index 7cc580718..c266dcda4 100644 --- a/diracx-cli/tests/test_login.py +++ b/diracx-cli/tests/test_login.py @@ -6,7 +6,7 @@ async def test_login(monkeypatch, capfd, cli_env): - from diracx.testing import test_login + from diracx.testing.utils import test_login return await test_login(monkeypatch, capfd, cli_env) diff --git a/diracx-client/src/diracx/client/__init__.py b/diracx-client/src/diracx/client/__init__.py index 1588c23a0..5470c74bd 100644 --- a/diracx-client/src/diracx/client/__init__.py +++ b/diracx-client/src/diracx/client/__init__.py @@ -1,15 +1,3 @@ from __future__ import absolute_import -__all__ = [ - "aio", - "models", - "sync", -] - -from typing import TYPE_CHECKING - - -if TYPE_CHECKING: - from . import aio - from . import models - from . import sync +__all__ = [] diff --git a/diracx-client/src/diracx/client/patches/auth/aio.py b/diracx-client/src/diracx/client/patches/auth/aio.py index a8a13fdb4..51016f677 100644 --- a/diracx-client/src/diracx/client/patches/auth/aio.py +++ b/diracx-client/src/diracx/client/patches/auth/aio.py @@ -14,7 +14,7 @@ from azure.core.pipeline import PipelineResponse from azure.core.tracing.decorator_async import distributed_trace_async -from diracx.core.models.auth import TokenResponse +from diracx.core.models import TokenResponse from ..._generated.aio.operations._operations import ( _models, diff --git a/diracx-client/src/diracx/client/patches/client/common.py b/diracx-client/src/diracx/client/patches/client/common.py index 3d59585fa..4f852fe45 100644 --- a/diracx-client/src/diracx/client/patches/client/common.py +++ b/diracx-client/src/diracx/client/patches/client/common.py @@ -20,7 +20,7 @@ import jwt from azure.core.credentials import AccessToken from diracx.core.utils import EXPIRES_GRACE_SECONDS, serialize_credentials -from diracx.core.models.auth import TokenResponse +from diracx.core.models import TokenResponse class TokenStatus(Enum): diff --git a/diracx-client/src/diracx/client/patches/jobs/common.py b/diracx-client/src/diracx/client/patches/jobs/common.py index 3fdf61182..597cda308 100644 --- a/diracx-client/src/diracx/client/patches/jobs/common.py +++ b/diracx-client/src/diracx/client/patches/jobs/common.py @@ -16,7 +16,7 @@ from io import BytesIO, IOBase from typing import Any, IO, Dict, TypedDict, Union, Unpack, cast, Literal -from diracx.core.models.search import SearchSpec +from diracx.core.models import SearchSpec class ResponseExtra(TypedDict, total=False): diff --git a/diracx-client/src/diracx/client/patches/utils.py b/diracx-client/src/diracx/client/patches/utils.py index 30584cabe..ebd811dda 100644 --- a/diracx-client/src/diracx/client/patches/utils.py +++ b/diracx-client/src/diracx/client/patches/utils.py @@ -22,7 +22,7 @@ from azure.core.pipeline import PipelineRequest from azure.core.pipeline.policies import BearerTokenCredentialPolicy -from diracx.core.models.auth import TokenResponse +from diracx.core.models import TokenResponse from diracx.core.preferences import DiracxPreferences, get_diracx_preferences diff --git a/diracx-client/tests/test_auth.py b/diracx-client/tests/test_auth.py index 7313cb299..e6aa6bee8 100644 --- a/diracx-client/tests/test_auth.py +++ b/diracx-client/tests/test_auth.py @@ -9,7 +9,7 @@ from azure.core.credentials import AccessToken from diracx.client.patches.utils import get_token -from diracx.core.models.auth import TokenResponse +from diracx.core.models import TokenResponse from diracx.core.utils import serialize_credentials # Create a fake jwt dictionary diff --git a/diracx-core/src/diracx/core/__init__.py b/diracx-core/src/diracx/core/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-core/src/diracx/core/__init__.py +++ b/diracx-core/src/diracx/core/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-core/src/diracx/core/config/__init__.py b/diracx-core/src/diracx/core/config/__init__.py index 37ff457ca..35d5fa4e9 100644 --- a/diracx-core/src/diracx/core/config/__init__.py +++ b/diracx-core/src/diracx/core/config/__init__.py @@ -2,20 +2,38 @@ from __future__ import annotations -from .schema import Config -from .sources import ( - ConfigSource, - ConfigSourceUrl, - LocalGitConfigSource, - RemoteGitConfigSource, - is_running_in_async_context, -) - -__all__ = ( +__all__ = [ "Config", "ConfigSource", "ConfigSourceUrl", + "DIRACConfig", + "GroupConfig", + "IdpConfig", "LocalGitConfigSource", + "OperationsConfig", + "RegistryConfig", "RemoteGitConfigSource", + "SerializableSet", + "SupportInfo", + "UserConfig", "is_running_in_async_context", +] + +from .schema import ( + Config, + DIRACConfig, + GroupConfig, + IdpConfig, + OperationsConfig, + RegistryConfig, + SerializableSet, + SupportInfo, + UserConfig, +) +from .sources import ( + ConfigSource, + ConfigSourceUrl, + LocalGitConfigSource, + RemoteGitConfigSource, + is_running_in_async_context, ) diff --git a/diracx-core/src/diracx/core/config/schema.py b/diracx-core/src/diracx/core/config/schema.py index 62c37b17c..cf76c3530 100644 --- a/diracx-core/src/diracx/core/config/schema.py +++ b/diracx-core/src/diracx/core/config/schema.py @@ -16,8 +16,8 @@ ) from pydantic.functional_serializers import PlainSerializer -from ..properties import SecurityProperty -from ..utils import recursive_merge +from diracx.core.properties import SecurityProperty +from diracx.core.utils import recursive_merge # By default the serialization of set doesn't have a well defined ordering so # we have to use a custom type to make sure the values are always sorted. diff --git a/diracx-core/src/diracx/core/config/sources.py b/diracx-core/src/diracx/core/config/sources.py index a11e8f18a..f16fffa82 100644 --- a/diracx-core/src/diracx/core/config/sources.py +++ b/diracx-core/src/diracx/core/config/sources.py @@ -20,9 +20,10 @@ from cachetools import Cache, LRUCache from pydantic import AnyUrl, BeforeValidator, TypeAdapter, UrlConstraints -from ..exceptions import BadConfigurationVersionError -from ..extensions import DiracEntryPoint, select_from_extension -from ..utils import TwoLevelCache +from diracx.core.exceptions import BadConfigurationVersionError +from diracx.core.extensions import DiracEntryPoint, select_from_extension +from diracx.core.utils import TwoLevelCache + from .schema import Config DEFAULT_CONFIG_FILE = "default.yml" diff --git a/diracx-core/src/diracx/core/exceptions.py b/diracx-core/src/diracx/core/exceptions.py index 4f45fc37b..a4d927393 100644 --- a/diracx-core/src/diracx/core/exceptions.py +++ b/diracx-core/src/diracx/core/exceptions.py @@ -1,5 +1,21 @@ from __future__ import annotations +__all__ = [ + "AuthorizationError", + "DiracError", + "DiracHttpResponseError", + "IAMClientError", + "IAMServerError", + "InvalidCredentialsError", + "InvalidQueryError", + "NotReadyError", + "PendingAuthorizationError", + "SandboxAlreadyAssignedError", + "SandboxAlreadyInsertedError", + "SandboxNotFoundError", + "TokenNotFoundError", +] + from http import HTTPStatus diff --git a/diracx-core/src/diracx/core/extensions.py b/diracx-core/src/diracx/core/extensions.py index a6bbe234b..4aeac7c15 100644 --- a/diracx-core/src/diracx/core/extensions.py +++ b/diracx-core/src/diracx/core/extensions.py @@ -1,10 +1,10 @@ from __future__ import annotations __all__ = [ + "DiracEntryPoint", "extensions_by_priority", "select_from_extension", "supports_extending", - "DiracEntryPoint", ] from collections import defaultdict diff --git a/diracx-core/src/diracx/core/models/__init__.py b/diracx-core/src/diracx/core/models/__init__.py index e69de29bb..ea0c1b3a6 100644 --- a/diracx-core/src/diracx/core/models/__init__.py +++ b/diracx-core/src/diracx/core/models/__init__.py @@ -0,0 +1,97 @@ +"""Models used to define the data structure of the requests and responses for the DiracX API. + +Shared between the client components (cli, api) and services components (db, logic, routers). +""" + +# in order to avoid DIRAC from failing to import TokenResponse +from __future__ import annotations + +__all__ = [ + "AccessTokenPayload", + "ChecksumAlgorithm", + "GrantType", + "GroupInfo", + "HeartbeatData", + "InitiateDeviceFlowResponse", + "InsertedJob", + "JobAttributes", + "JobCommand", + "JobLoggingRecord", + "JobMetaData", + "JobMinorStatus", + "JobParameters", + "JobStatus", + "JobStatusReturn", + "JobStatusUpdate", + "Metadata", + "OpenIDConfiguration", + "RefreshTokenPayload", + "ReplicaMap", + "SandboxDownloadResponse", + "SandboxFormat", + "SandboxInfo", + "SandboxType", + "SandboxUploadResponse", + "ScalarSearchOperator", + "ScalarSearchSpec", + "SearchParams", + "SearchSpec", + "SetJobStatusReturn", + "SortDirection", + "SortSpec", + "SummaryParams", + "TokenPayload", + "TokenResponse", + "TokenTypeHint", + "UserInfo", + "VectorSearchOperator", + "VectorSearchSpec", +] + +from .auth import ( + AccessTokenPayload, + GrantType, + GroupInfo, + InitiateDeviceFlowResponse, + Metadata, + OpenIDConfiguration, + RefreshTokenPayload, + TokenPayload, + TokenResponse, + TokenTypeHint, + UserInfo, +) +from .job import ( + HeartbeatData, + InsertedJob, + JobAttributes, + JobCommand, + JobLoggingRecord, + JobMetaData, + JobMinorStatus, + JobParameters, + JobStatus, + JobStatusReturn, + JobStatusUpdate, + SetJobStatusReturn, +) +from .replica_map import ReplicaMap +from .sandbox import ( + ChecksumAlgorithm, + SandboxDownloadResponse, + SandboxFormat, + SandboxInfo, + SandboxType, + SandboxUploadResponse, +) +from .search import ( + ScalarSearchOperator, + ScalarSearchSpec, + SearchParams, + SearchSpec, + SortDirection, + SortSpec, + SummaryParams, + VectorSearchOperator, + VectorSearchSpec, +) diff --git a/diracx-core/src/diracx/core/preferences.py b/diracx-core/src/diracx/core/preferences.py index 6df6f7104..acecfb9bb 100644 --- a/diracx-core/src/diracx/core/preferences.py +++ b/diracx-core/src/diracx/core/preferences.py @@ -1,9 +1,10 @@ from __future__ import annotations -from pydantic import field_validator -from pydantic_settings import SettingsConfigDict - -__all__ = ("DiracxPreferences", "OutputFormats", "get_diracx_preferences") +__all__ = [ + "DiracxPreferences", + "OutputFormats", + "get_diracx_preferences", +] import logging import sys @@ -11,8 +12,8 @@ from functools import lru_cache from pathlib import Path -from pydantic import AnyHttpUrl, Field -from pydantic_settings import BaseSettings +from pydantic import AnyHttpUrl, Field, field_validator +from pydantic_settings import BaseSettings, SettingsConfigDict from .utils import dotenv_files_from_environment diff --git a/diracx-core/src/diracx/core/properties.py b/diracx-core/src/diracx/core/properties.py index cdb664c76..32e07b909 100644 --- a/diracx-core/src/diracx/core/properties.py +++ b/diracx-core/src/diracx/core/properties.py @@ -2,6 +2,35 @@ from __future__ import annotations +__all__ = [ + # Properties + "ACCOUNTING_MONITOR", + "ALARMS_MANAGEMENT", + "BOOKKEEPING_MANAGEMENT", + "CS_ADMINISTRATOR", + "FC_MANAGEMENT", + "FULL_DELEGATION", + "GENERIC_PILOT", + "JOB_ADMINISTRATOR", + "JOB_MONITOR", + "JOB_SHARING", + "LIMITED_DELEGATION", + "NORMAL_USER", + "OPERATOR", + "PILOT", + "PPG_AUTHORITY", + "PRIVATE_LIMITED_DELEGATION", + "PRODUCTION_MANAGEMENT", + "PROXY_MANAGEMENT", + "SERVICE_ADMINISTRATOR", + "SITE_MANAGER", + "TRUSTED_HOST", + "USER_MANAGER", + # Classes + "SecurityProperty", + "UnevaluatedProperty", +] + import inspect import operator from collections.abc import Callable @@ -10,7 +39,7 @@ from pydantic import GetCoreSchemaHandler from pydantic_core import CoreSchema, core_schema -from diracx.core.extensions import select_from_extension +from .extensions import select_from_extension class SecurityProperty(str): diff --git a/diracx-core/src/diracx/core/resources.py b/diracx-core/src/diracx/core/resources.py index 4ec3f83e0..55ce03a5b 100644 --- a/diracx-core/src/diracx/core/resources.py +++ b/diracx-core/src/diracx/core/resources.py @@ -6,8 +6,8 @@ from DIRACCommon.ConfigurationSystem.Client.Helpers.Resources import getDIRACPlatform from DIRACCommon.Core.Utilities.ReturnValues import returnValueOrRaise -from diracx.core.config import Config -from diracx.core.extensions import DiracEntryPoint, supports_extending +from .config import Config +from .extensions import DiracEntryPoint, supports_extending @supports_extending(DiracEntryPoint.RESOURCES, "find_compatible_platforms") diff --git a/diracx-core/src/diracx/core/s3.py b/diracx-core/src/diracx/core/s3.py index afc0c9d20..da37c6999 100644 --- a/diracx-core/src/diracx/core/s3.py +++ b/diracx-core/src/diracx/core/s3.py @@ -2,12 +2,13 @@ from __future__ import annotations -__all__ = ( +__all__ = [ + "b16_to_b64", + "generate_presigned_upload", "s3_bucket_exists", "s3_bulk_delete_with_retry", "s3_object_exists", - "generate_presigned_upload", -) +] import asyncio import base64 @@ -15,7 +16,7 @@ from botocore.errorfactory import ClientError -from .models.sandbox import ChecksumAlgorithm +from diracx.core.models import ChecksumAlgorithm if TYPE_CHECKING: from typing import TypedDict diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index 40c75afd2..dd55a88a8 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -2,18 +2,18 @@ from __future__ import annotations -import json - -from diracx.core.properties import SecurityProperty -from diracx.core.s3 import s3_bucket_exists - -__all__ = ( - "SqlalchemyDsn", +__all__ = [ + "AuthSettings", + "DevelopmentSettings", "LocalFileUrl", + "SandboxStoreSettings", "ServiceSettingsBase", -) + "SqlalchemyDsn", + "TokenSigningKeyStore", +] import contextlib +import json from collections.abc import AsyncIterator from pathlib import Path from typing import TYPE_CHECKING, Annotated, Any, Self, TypeVar, cast @@ -36,6 +36,9 @@ ) from pydantic_settings import BaseSettings, SettingsConfigDict +from .properties import SecurityProperty +from .s3 import s3_bucket_exists + if TYPE_CHECKING: from types_aiobotocore_s3.client import S3Client diff --git a/diracx-core/src/diracx/core/utils.py b/diracx-core/src/diracx/core/utils.py index c0e2b0c6a..8949d07f3 100644 --- a/diracx-core/src/diracx/core/utils.py +++ b/diracx-core/src/diracx/core/utils.py @@ -1,13 +1,14 @@ from __future__ import annotations __all__ = [ - "dotenv_files_from_environment", - "serialize_credentials", - "read_credentials", - "write_credentials", + "EXPIRES_GRACE_SECONDS", "TwoLevelCache", "batched_async", + "dotenv_files_from_environment", + "read_credentials", "recursive_merge", + "serialize_credentials", + "write_credentials", ] import fcntl @@ -26,8 +27,8 @@ from cachetools import Cache, TTLCache -from diracx.core.exceptions import NotReadyError -from diracx.core.models.auth import TokenResponse +from .exceptions import NotReadyError +from .models import TokenResponse logger = logging.getLogger(__name__) diff --git a/diracx-core/tests/test_config_source.py b/diracx-core/tests/test_config_source.py index fa61232e2..e1a095aff 100644 --- a/diracx-core/tests/test_config_source.py +++ b/diracx-core/tests/test_config_source.py @@ -5,8 +5,7 @@ import pytest -from diracx.core.config import ConfigSource, RemoteGitConfigSource -from diracx.core.config.schema import Config +from diracx.core.config import Config, ConfigSource, RemoteGitConfigSource # The diracx-chart contains a CS example TEST_REPO = "git+https://github.com/DIRACGrid/diracx-charts.git" diff --git a/diracx-core/tests/test_replica_map.py b/diracx-core/tests/test_replica_map.py index aca6e6d79..bc5037bb4 100644 --- a/diracx-core/tests/test_replica_map.py +++ b/diracx-core/tests/test_replica_map.py @@ -2,8 +2,8 @@ import pytest +from diracx.core.models import ReplicaMap from diracx.core.models.replica_map import ( - ReplicaMap, _validate_adler32, _validate_guid, _validate_lfn, diff --git a/diracx-core/tests/test_schema_legacy_adaptor.py b/diracx-core/tests/test_schema_legacy_adaptor.py index 7b2d61bfe..206e5e1b3 100644 --- a/diracx-core/tests/test_schema_legacy_adaptor.py +++ b/diracx-core/tests/test_schema_legacy_adaptor.py @@ -7,7 +7,8 @@ import pytest from pydantic import ValidationError -from diracx.core.config.schema import BaseModel, SerializableSet +from diracx.core.config import SerializableSet +from diracx.core.config.schema import BaseModel from diracx.core.properties import NORMAL_USER, PRODUCTION_MANAGEMENT, SecurityProperty diff --git a/diracx-core/tests/test_utils.py b/diracx-core/tests/test_utils.py index 5b12dfe52..b88483e39 100644 --- a/diracx-core/tests/test_utils.py +++ b/diracx-core/tests/test_utils.py @@ -9,7 +9,7 @@ import pytest from diracx.core.exceptions import NotReadyError -from diracx.core.models.auth import TokenResponse +from diracx.core.models import TokenResponse from diracx.core.utils import ( TwoLevelCache, dotenv_files_from_environment, diff --git a/diracx-db/src/diracx/db/__init__.py b/diracx-db/src/diracx/db/__init__.py index 824eb5624..30c2cb0e4 100644 --- a/diracx-db/src/diracx/db/__init__.py +++ b/diracx-db/src/diracx/db/__init__.py @@ -1,5 +1,5 @@ from __future__ import annotations -__all__ = ("sql", "os", "exceptions") +__all__ = ["exceptions", "os", "sql"] from . import exceptions, os, sql diff --git a/diracx-db/src/diracx/db/exceptions.py b/diracx-db/src/diracx/db/exceptions.py index 36a3e278f..569549908 100644 --- a/diracx-db/src/diracx/db/exceptions.py +++ b/diracx-db/src/diracx/db/exceptions.py @@ -1,5 +1,7 @@ from __future__ import annotations +__all__ = ["DBUnavailableError"] + class DBUnavailableError(Exception): pass diff --git a/diracx-db/src/diracx/db/os/__init__.py b/diracx-db/src/diracx/db/os/__init__.py index 535e2a954..8da49758a 100644 --- a/diracx-db/src/diracx/db/os/__init__.py +++ b/diracx-db/src/diracx/db/os/__init__.py @@ -1,5 +1,7 @@ from __future__ import annotations -__all__ = ("JobParametersDB",) +__all__ = [ + "JobParametersDB", +] from .job_parameters import JobParametersDB diff --git a/diracx-db/src/diracx/db/os/utils.py b/diracx-db/src/diracx/db/os/utils.py index 80d87a8fd..aae2bb4c6 100644 --- a/diracx-db/src/diracx/db/os/utils.py +++ b/diracx-db/src/diracx/db/os/utils.py @@ -1,6 +1,6 @@ from __future__ import annotations -__all__ = ("BaseOSDB",) +__all__ = ["BaseOSDB"] import contextlib import json diff --git a/diracx-db/src/diracx/db/sql/__init__.py b/diracx-db/src/diracx/db/sql/__init__.py index c38db7eba..704dce3bc 100644 --- a/diracx-db/src/diracx/db/sql/__init__.py +++ b/diracx-db/src/diracx/db/sql/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations -__all__ = ( +__all__ = [ "AuthDB", "JobDB", "JobLoggingDB", @@ -8,7 +8,7 @@ "ResourceStatusDB", "SandboxMetadataDB", "TaskQueueDB", -) +] from .auth.db import AuthDB from .job.db import JobDB diff --git a/diracx-db/src/diracx/db/sql/auth/__init__.py b/diracx-db/src/diracx/db/sql/auth/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-db/src/diracx/db/sql/auth/__init__.py +++ b/diracx-db/src/diracx/db/sql/auth/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-db/src/diracx/db/sql/dummy/__init__.py b/diracx-db/src/diracx/db/sql/dummy/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-db/src/diracx/db/sql/dummy/__init__.py +++ b/diracx-db/src/diracx/db/sql/dummy/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-db/src/diracx/db/sql/job/__init__.py b/diracx-db/src/diracx/db/sql/job/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-db/src/diracx/db/sql/job/__init__.py +++ b/diracx-db/src/diracx/db/sql/job/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-db/src/diracx/db/sql/job/db.py b/diracx-db/src/diracx/db/sql/job/db.py index bb28aa5cf..14a287877 100644 --- a/diracx-db/src/diracx/db/sql/job/db.py +++ b/diracx-db/src/diracx/db/sql/job/db.py @@ -12,8 +12,7 @@ from sqlalchemy.sql import expression from diracx.core.exceptions import InvalidQueryError -from diracx.core.models.job import JobCommand -from diracx.core.models.search import SearchSpec, SortSpec +from diracx.core.models import JobCommand, SearchSpec, SortSpec from ..utils import BaseSQLDB, _get_columns from ..utils.functions import utcnow diff --git a/diracx-db/src/diracx/db/sql/job_logging/__init__.py b/diracx-db/src/diracx/db/sql/job_logging/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-db/src/diracx/db/sql/job_logging/__init__.py +++ b/diracx-db/src/diracx/db/sql/job_logging/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-db/src/diracx/db/sql/job_logging/db.py b/diracx-db/src/diracx/db/sql/job_logging/db.py index 8f42e157a..b00990988 100644 --- a/diracx-db/src/diracx/db/sql/job_logging/db.py +++ b/diracx-db/src/diracx/db/sql/job_logging/db.py @@ -6,7 +6,7 @@ from sqlalchemy import delete, func, insert, select -from diracx.core.models.job import JobLoggingRecord, JobStatusReturn +from diracx.core.models import JobLoggingRecord, JobStatusReturn from ..utils import BaseSQLDB from .schema import JobLoggingDBBase, LoggingInfo diff --git a/diracx-db/src/diracx/db/sql/pilot_agents/__init__.py b/diracx-db/src/diracx/db/sql/pilot_agents/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-db/src/diracx/db/sql/pilot_agents/__init__.py +++ b/diracx-db/src/diracx/db/sql/pilot_agents/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-db/src/diracx/db/sql/rss/__init__.py b/diracx-db/src/diracx/db/sql/rss/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-db/src/diracx/db/sql/rss/__init__.py +++ b/diracx-db/src/diracx/db/sql/rss/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-db/src/diracx/db/sql/sandbox_metadata/__init__.py b/diracx-db/src/diracx/db/sql/sandbox_metadata/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-db/src/diracx/db/sql/sandbox_metadata/__init__.py +++ b/diracx-db/src/diracx/db/sql/sandbox_metadata/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-db/src/diracx/db/sql/sandbox_metadata/db.py b/diracx-db/src/diracx/db/sql/sandbox_metadata/db.py index d81a92916..291a04bff 100644 --- a/diracx-db/src/diracx/db/sql/sandbox_metadata/db.py +++ b/diracx-db/src/diracx/db/sql/sandbox_metadata/db.py @@ -20,8 +20,7 @@ SandboxAlreadyInsertedError, SandboxNotFoundError, ) -from diracx.core.models.auth import UserInfo -from diracx.core.models.sandbox import SandboxInfo, SandboxType +from diracx.core.models import SandboxInfo, SandboxType, UserInfo from diracx.db.sql.utils.base import BaseSQLDB from diracx.db.sql.utils.functions import substract_date, utcnow diff --git a/diracx-db/src/diracx/db/sql/task_queue/__init__.py b/diracx-db/src/diracx/db/sql/task_queue/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-db/src/diracx/db/sql/task_queue/__init__.py +++ b/diracx-db/src/diracx/db/sql/task_queue/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-db/src/diracx/db/sql/utils/__init__.py b/diracx-db/src/diracx/db/sql/utils/__init__.py index ad262186b..2abd93fac 100644 --- a/diracx-db/src/diracx/db/sql/utils/__init__.py +++ b/diracx-db/src/diracx/db/sql/utils/__init__.py @@ -1,25 +1,25 @@ from __future__ import annotations __all__ = [ - "_get_columns", - "utcnow", "BaseSQLDB", "EnumBackedBool", - "enum_column", + "SQLDBUnavailableError", + "_get_columns", "apply_search_filters", "apply_sort_constraints", - "substract_date", - "hash", - "SQLDBUnavailableError", - "uuid7_from_datetime", - "uuid7_to_datetime", "datetime_now", + "enum_column", + "hash", "str32", "str64", "str128", "str255", "str512", "str1024", + "substract_date", + "utcnow", + "uuid7_from_datetime", + "uuid7_to_datetime", ] from .base import ( diff --git a/diracx-db/src/diracx/db/sql/utils/base.py b/diracx-db/src/diracx/db/sql/utils/base.py index e3fcff56d..4a8ddfd65 100644 --- a/diracx-db/src/diracx/db/sql/utils/base.py +++ b/diracx-db/src/diracx/db/sql/utils/base.py @@ -21,7 +21,7 @@ from diracx.core.exceptions import InvalidQueryError from diracx.core.extensions import DiracEntryPoint, select_from_extension -from diracx.core.models.search import ( +from diracx.core.models import ( ScalarSearchOperator, SearchSpec, SortDirection, diff --git a/diracx-db/tests/auth/__init__.py b/diracx-db/tests/auth/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-db/tests/auth/__init__.py +++ b/diracx-db/tests/auth/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-db/tests/jobs/__init__.py b/diracx-db/tests/jobs/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-db/tests/jobs/__init__.py +++ b/diracx-db/tests/jobs/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-db/tests/jobs/test_job_db.py b/diracx-db/tests/jobs/test_job_db.py index 2bb53a2a5..a42373475 100644 --- a/diracx-db/tests/jobs/test_job_db.py +++ b/diracx-db/tests/jobs/test_job_db.py @@ -8,7 +8,7 @@ from sqlalchemy.exc import IntegrityError from diracx.core.exceptions import InvalidQueryError -from diracx.core.models.search import ( +from diracx.core.models import ( ScalarSearchOperator, ScalarSearchSpec, SortDirection, diff --git a/diracx-db/tests/jobs/test_job_logging_db.py b/diracx-db/tests/jobs/test_job_logging_db.py index 6465eda89..a45457525 100644 --- a/diracx-db/tests/jobs/test_job_logging_db.py +++ b/diracx-db/tests/jobs/test_job_logging_db.py @@ -4,7 +4,7 @@ import pytest -from diracx.core.models.job import JobLoggingRecord, JobStatus +from diracx.core.models import JobLoggingRecord, JobStatus from diracx.db.sql import JobLoggingDB diff --git a/diracx-db/tests/jobs/test_sandbox_metadata.py b/diracx-db/tests/jobs/test_sandbox_metadata.py index eb6603e57..290c523a7 100644 --- a/diracx-db/tests/jobs/test_sandbox_metadata.py +++ b/diracx-db/tests/jobs/test_sandbox_metadata.py @@ -8,8 +8,7 @@ import sqlalchemy from diracx.core.exceptions import SandboxAlreadyInsertedError, SandboxNotFoundError -from diracx.core.models.auth import UserInfo -from diracx.core.models.sandbox import SandboxInfo +from diracx.core.models import SandboxInfo, UserInfo from diracx.db.sql.sandbox_metadata.db import SandboxMetadataDB from diracx.db.sql.sandbox_metadata.schema import SandBoxes, SBEntityMapping diff --git a/diracx-db/tests/opensearch/__init__.py b/diracx-db/tests/opensearch/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-db/tests/opensearch/__init__.py +++ b/diracx-db/tests/opensearch/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-db/tests/pilot_agents/__init__.py b/diracx-db/tests/pilot_agents/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-db/tests/pilot_agents/__init__.py +++ b/diracx-db/tests/pilot_agents/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-db/tests/rss/__init__.py b/diracx-db/tests/rss/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-db/tests/rss/__init__.py +++ b/diracx-db/tests/rss/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-db/tests/utils/__init__.py b/diracx-db/tests/utils/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-db/tests/utils/__init__.py +++ b/diracx-db/tests/utils/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-logic/src/diracx/logic/__init__.py b/diracx-logic/src/diracx/logic/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-logic/src/diracx/logic/__init__.py +++ b/diracx-logic/src/diracx/logic/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-logic/src/diracx/logic/auth/__init__.py b/diracx-logic/src/diracx/logic/auth/__init__.py index e69de29bb..c6ef415e4 100644 --- a/diracx-logic/src/diracx/logic/auth/__init__.py +++ b/diracx-logic/src/diracx/logic/auth/__init__.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +__all__ = [ + "complete_authorization_flow", + "create_token", + "decrypt_state", + "do_device_flow", + "encrypt_state", + "finish_device_flow", + "get_installation_metadata", + "get_jwks", + "get_oidc_token", + "get_openid_configuration", + "get_refresh_tokens", + "get_server_metadata", + "initiate_authorization_flow", + "initiate_device_flow", + "parse_and_validate_scope", + "perform_legacy_exchange", + "read_token", + "revoke_refresh_token_by_jti", + "revoke_refresh_token_by_refresh_token", + "verify_dirac_refresh_token", +] + +from .authorize_code_flow import ( + complete_authorization_flow, + initiate_authorization_flow, +) +from .device_flow import do_device_flow, finish_device_flow, initiate_device_flow +from .management import ( + get_refresh_tokens, + revoke_refresh_token_by_jti, + revoke_refresh_token_by_refresh_token, +) +from .token import create_token, get_oidc_token, perform_legacy_exchange +from .utils import ( + decrypt_state, + encrypt_state, + get_server_metadata, + parse_and_validate_scope, + read_token, + verify_dirac_refresh_token, +) +from .well_known import get_installation_metadata, get_jwks, get_openid_configuration diff --git a/diracx-logic/src/diracx/logic/auth/authorize_code_flow.py b/diracx-logic/src/diracx/logic/auth/authorize_code_flow.py index f65609eb3..5c43fdfda 100644 --- a/diracx-logic/src/diracx/logic/auth/authorize_code_flow.py +++ b/diracx-logic/src/diracx/logic/auth/authorize_code_flow.py @@ -5,7 +5,7 @@ from typing import Literal from diracx.core.config import Config -from diracx.core.models.auth import GrantType +from diracx.core.models import GrantType from diracx.core.properties import SecurityProperty from diracx.core.settings import AuthSettings from diracx.db.sql import AuthDB diff --git a/diracx-logic/src/diracx/logic/auth/device_flow.py b/diracx-logic/src/diracx/logic/auth/device_flow.py index 23951596e..39be694da 100644 --- a/diracx-logic/src/diracx/logic/auth/device_flow.py +++ b/diracx-logic/src/diracx/logic/auth/device_flow.py @@ -3,7 +3,7 @@ from __future__ import annotations from diracx.core.config import Config -from diracx.core.models.auth import GrantType, InitiateDeviceFlowResponse +from diracx.core.models import GrantType, InitiateDeviceFlowResponse from diracx.core.properties import SecurityProperty from diracx.core.settings import AuthSettings from diracx.db.sql import AuthDB diff --git a/diracx-logic/src/diracx/logic/auth/management.py b/diracx-logic/src/diracx/logic/auth/management.py index 57027ad4f..4dfab8ce6 100644 --- a/diracx-logic/src/diracx/logic/auth/management.py +++ b/diracx-logic/src/diracx/logic/auth/management.py @@ -7,10 +7,11 @@ from uuid_utils import UUID from diracx.core.exceptions import InvalidCredentialsError -from diracx.core.models.auth import TokenTypeHint +from diracx.core.models import TokenTypeHint from diracx.core.settings import AuthSettings from diracx.db.sql import AuthDB -from diracx.logic.auth.utils import verify_dirac_refresh_token + +from .utils import verify_dirac_refresh_token logger = logging.getLogger(__name__) diff --git a/diracx-logic/src/diracx/logic/auth/token.py b/diracx-logic/src/diracx/logic/auth/token.py index 40784db9c..d5f8908c5 100644 --- a/diracx-logic/src/diracx/logic/auth/token.py +++ b/diracx-logic/src/diracx/logic/auth/token.py @@ -18,7 +18,7 @@ InvalidCredentialsError, PendingAuthorizationError, ) -from diracx.core.models.auth import ( +from diracx.core.models import ( AccessTokenPayload, GrantType, RefreshTokenPayload, diff --git a/diracx-logic/src/diracx/logic/auth/utils.py b/diracx-logic/src/diracx/logic/auth/utils.py index 113de9f8c..35e6db424 100644 --- a/diracx-logic/src/diracx/logic/auth/utils.py +++ b/diracx-logic/src/diracx/logic/auth/utils.py @@ -14,9 +14,9 @@ from typing_extensions import TypedDict from uuid_utils import UUID -from diracx.core.config.schema import Config +from diracx.core.config import Config from diracx.core.exceptions import AuthorizationError, IAMClientError, IAMServerError -from diracx.core.models.auth import GrantType +from diracx.core.models import GrantType from diracx.core.properties import SecurityProperty from diracx.core.settings import AuthSettings diff --git a/diracx-logic/src/diracx/logic/auth/well_known.py b/diracx-logic/src/diracx/logic/auth/well_known.py index 2c53c2223..0e2519cde 100644 --- a/diracx-logic/src/diracx/logic/auth/well_known.py +++ b/diracx-logic/src/diracx/logic/auth/well_known.py @@ -1,7 +1,7 @@ from __future__ import annotations -from diracx.core.config.schema import Config -from diracx.core.models.auth import GroupInfo, Metadata, OpenIDConfiguration +from diracx.core.config import Config +from diracx.core.models import GroupInfo, Metadata, OpenIDConfiguration from diracx.core.settings import AuthSettings diff --git a/diracx-logic/src/diracx/logic/jobs/__init__.py b/diracx-logic/src/diracx/logic/jobs/__init__.py index e69de29bb..f8a0a7ab9 100644 --- a/diracx-logic/src/diracx/logic/jobs/__init__.py +++ b/diracx-logic/src/diracx/logic/jobs/__init__.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +__all__ = [ + "MAX_PER_PAGE", + "SANDBOX_PFN_REGEX", + "add_heartbeat", + "assign_sandbox_to_job", + "check_and_prepare_job", + "clean_sandboxes", + "get_job_commands", + "get_job_sandbox", + "get_job_sandboxes", + "get_sandbox_file", + "initiate_sandbox_upload", + "make_job_manifest_config", + "remove_jobs", + "remove_jobs_from_task_queue", + "reschedule_jobs", + "search", + "set_job_parameters_or_attributes", + "set_job_statuses", + "submit_jdl_jobs", + "summary", + "unassign_jobs_sandboxes", +] + +from .query import MAX_PER_PAGE, search, summary +from .sandboxes import ( + SANDBOX_PFN_REGEX, + assign_sandbox_to_job, + clean_sandboxes, + get_job_sandbox, + get_job_sandboxes, + get_sandbox_file, + initiate_sandbox_upload, + unassign_jobs_sandboxes, +) +from .status import ( + add_heartbeat, + get_job_commands, + remove_jobs, + remove_jobs_from_task_queue, + reschedule_jobs, + set_job_parameters_or_attributes, + set_job_statuses, +) +from .submission import submit_jdl_jobs +from .utils import check_and_prepare_job, make_job_manifest_config diff --git a/diracx-logic/src/diracx/logic/jobs/query.py b/diracx-logic/src/diracx/logic/jobs/query.py index a3e1a9732..91c27fb1c 100644 --- a/diracx-logic/src/diracx/logic/jobs/query.py +++ b/diracx-logic/src/diracx/logic/jobs/query.py @@ -3,8 +3,8 @@ import logging from typing import Any -from diracx.core.config.schema import Config -from diracx.core.models.search import ( +from diracx.core.config import Config +from diracx.core.models import ( ScalarSearchOperator, SearchParams, SummaryParams, diff --git a/diracx-logic/src/diracx/logic/jobs/sandboxes.py b/diracx-logic/src/diracx/logic/jobs/sandboxes.py index 3bfd1fdd1..7d4b6bb9b 100644 --- a/diracx-logic/src/diracx/logic/jobs/sandboxes.py +++ b/diracx-logic/src/diracx/logic/jobs/sandboxes.py @@ -6,12 +6,12 @@ from typing import TYPE_CHECKING, Any, Literal from diracx.core.exceptions import SandboxAlreadyInsertedError, SandboxNotFoundError -from diracx.core.models.auth import UserInfo -from diracx.core.models.sandbox import ( +from diracx.core.models import ( SandboxDownloadResponse, SandboxInfo, SandboxType, SandboxUploadResponse, + UserInfo, ) from diracx.core.s3 import ( generate_presigned_upload, diff --git a/diracx-logic/src/diracx/logic/jobs/status.py b/diracx-logic/src/diracx/logic/jobs/status.py index aea40c34a..e337c293e 100644 --- a/diracx-logic/src/diracx/logic/jobs/status.py +++ b/diracx-logic/src/diracx/logic/jobs/status.py @@ -1,15 +1,5 @@ from __future__ import annotations -__all__ = [ - "remove_jobs", - "set_job_statuses", - "reschedule_jobs", - "remove_jobs_from_task_queue", - "set_job_parameters_or_attributes", - "add_heartbeat", - "get_job_commands", -] - import logging from asyncio import TaskGroup from collections import defaultdict @@ -28,8 +18,8 @@ getStartAndEndTime, ) -from diracx.core.config.schema import Config -from diracx.core.models.job import ( +from diracx.core.config import Config +from diracx.core.models import ( HeartbeatData, JobAttributes, JobCommand, @@ -40,16 +30,18 @@ JobStatus, JobStatusUpdate, SetJobStatusReturn, + VectorSearchOperator, + VectorSearchSpec, ) -from diracx.core.models.search import VectorSearchOperator, VectorSearchSpec from diracx.db.os.job_parameters import JobParametersDB from diracx.db.sql.job.db import JobDB from diracx.db.sql.job_logging.db import JobLoggingDB from diracx.db.sql.sandbox_metadata.db import SandboxMetadataDB from diracx.db.sql.task_queue.db import TaskQueueDB from diracx.db.sql.utils.functions import utcnow -from diracx.logic.jobs.utils import check_and_prepare_job -from diracx.logic.task_queues.priority import recalculate_tq_shares_for_entity +from diracx.logic.task_queues import recalculate_tq_shares_for_entity + +from .utils import check_and_prepare_job logger = logging.getLogger(__name__) diff --git a/diracx-logic/src/diracx/logic/jobs/submission.py b/diracx-logic/src/diracx/logic/jobs/submission.py index 77018d64a..ff4be0fde 100644 --- a/diracx-logic/src/diracx/logic/jobs/submission.py +++ b/diracx-logic/src/diracx/logic/jobs/submission.py @@ -19,15 +19,16 @@ from pydantic import BaseModel from diracx.core.config import Config -from diracx.core.models.auth import UserInfo -from diracx.core.models.job import ( +from diracx.core.models import ( InsertedJob, JobLoggingRecord, JobStatus, + UserInfo, ) from diracx.db.sql.job.db import JobDB from diracx.db.sql.job_logging.db import JobLoggingDB -from diracx.logic.jobs.utils import check_and_prepare_job, make_job_manifest_config + +from .utils import check_and_prepare_job, make_job_manifest_config logger = logging.getLogger(__name__) diff --git a/diracx-logic/src/diracx/logic/rss/__init__.py b/diracx-logic/src/diracx/logic/rss/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-logic/src/diracx/logic/rss/__init__.py +++ b/diracx-logic/src/diracx/logic/rss/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-logic/src/diracx/logic/task_queues/__init__.py b/diracx-logic/src/diracx/logic/task_queues/__init__.py index e69de29bb..89395bb5f 100644 --- a/diracx-logic/src/diracx/logic/task_queues/__init__.py +++ b/diracx-logic/src/diracx/logic/task_queues/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +__all__ = ["recalculate_tq_shares_for_entity"] + +from .priority import recalculate_tq_shares_for_entity diff --git a/diracx-logic/src/diracx/logic/task_queues/priority.py b/diracx-logic/src/diracx/logic/task_queues/priority.py index 11500a5be..3f07eddce 100644 --- a/diracx-logic/src/diracx/logic/task_queues/priority.py +++ b/diracx-logic/src/diracx/logic/task_queues/priority.py @@ -3,7 +3,7 @@ from collections import defaultdict from typing import Any -from diracx.core.config.schema import Config +from diracx.core.config import Config from diracx.core.properties import JOB_SHARING from diracx.db.sql.task_queue.db import TaskQueueDB diff --git a/diracx-logic/tests/jobs/__init__.py b/diracx-logic/tests/jobs/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-logic/tests/jobs/__init__.py +++ b/diracx-logic/tests/jobs/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-logic/tests/jobs/test_sandboxes.py b/diracx-logic/tests/jobs/test_sandboxes.py index 6dc59197d..c4b11cc43 100644 --- a/diracx-logic/tests/jobs/test_sandboxes.py +++ b/diracx-logic/tests/jobs/test_sandboxes.py @@ -13,11 +13,10 @@ import sqlalchemy from diracx.core.exceptions import SandboxNotFoundError -from diracx.core.models.auth import UserInfo -from diracx.core.models.sandbox import ChecksumAlgorithm, SandboxFormat, SandboxInfo +from diracx.core.models import ChecksumAlgorithm, SandboxFormat, SandboxInfo, UserInfo from diracx.core.settings import SandboxStoreSettings from diracx.db.sql.sandbox_metadata.db import SandboxMetadataDB -from diracx.logic.jobs.sandboxes import ( +from diracx.logic.jobs import ( clean_sandboxes, get_sandbox_file, initiate_sandbox_upload, diff --git a/diracx-logic/tests/jobs/test_status.py b/diracx-logic/tests/jobs/test_status.py index b0e1f050f..f84b988e8 100644 --- a/diracx-logic/tests/jobs/test_status.py +++ b/diracx-logic/tests/jobs/test_status.py @@ -6,10 +6,10 @@ import pytest import sqlalchemy -from diracx.core.models.job import JobMetaData +from diracx.core.models import JobMetaData from diracx.db.os.job_parameters import JobParametersDB as RealJobParametersDB from diracx.db.sql.job.db import JobDB -from diracx.logic.jobs.status import set_job_parameters_or_attributes +from diracx.logic.jobs import set_job_parameters_or_attributes from diracx.testing.mock_osdb import MockOSDBMixin from diracx.testing.time import mock_sqlite_time diff --git a/diracx-logic/tests/rss/__init__.py b/diracx-logic/tests/rss/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-logic/tests/rss/__init__.py +++ b/diracx-logic/tests/rss/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-routers/pyproject.toml b/diracx-routers/pyproject.toml index 6cdb993b6..d798b1696 100644 --- a/diracx-routers/pyproject.toml +++ b/diracx-routers/pyproject.toml @@ -54,7 +54,7 @@ sandbox = "diracx.routers.jobs.access_policies:SandboxAccessPolicy" # Minimum version of the client supported [project.entry-points."diracx.min_client_version"] -diracx = "diracx.routers:DIRACX_MIN_CLIENT_VERSION" +diracx = "diracx.routers.factory:DIRACX_MIN_CLIENT_VERSION" [build-system] requires = ["hatchling", "hatch-vcs"] diff --git a/diracx-routers/src/diracx/routers/__init__.py b/diracx-routers/src/diracx/routers/__init__.py index 17785560c..33ce21a96 100644 --- a/diracx-routers/src/diracx/routers/__init__.py +++ b/diracx-routers/src/diracx/routers/__init__.py @@ -7,6 +7,7 @@ from __future__ import annotations -from .factory import DIRACX_MIN_CLIENT_VERSION, create_app, create_app_inner +# Special case due to a dependency in diracx-charts and DIRAC +__all__ = ["create_app"] -__all__ = ("create_app", "create_app_inner", "DIRACX_MIN_CLIENT_VERSION") +from .factory import create_app diff --git a/diracx-routers/src/diracx/routers/access_policies.py b/diracx-routers/src/diracx/routers/access_policies.py index e6f8019dd..c2e498d74 100644 --- a/diracx-routers/src/diracx/routers/access_policies.py +++ b/diracx-routers/src/diracx/routers/access_policies.py @@ -17,6 +17,10 @@ """ +from __future__ import annotations + +__all__ = ["BaseAccessPolicy", "check_permissions", "open_access"] + import functools import os import time @@ -27,21 +31,13 @@ from fastapi import Depends from diracx.core.extensions import DiracEntryPoint, select_from_extension -from diracx.core.models.auth import ( +from diracx.core.models import ( AccessTokenPayload, RefreshTokenPayload, ) from diracx.core.settings import DevelopmentSettings from diracx.routers.dependencies import auto_inject -from diracx.routers.utils.users import AuthorizedUserInfo, verify_dirac_access_token - -if "annotations" in globals(): - raise NotImplementedError( - "FastAPI bug: We normally would use `from __future__ import annotations` " - "but a bug in FastAPI prevents us from doing so " - "https://github.com/tiangolo/fastapi/pull/11355 " - "Until it is merged, we can work around it by using strings." - ) +from diracx.routers.utils import AuthorizedUserInfo, verify_dirac_access_token class BaseAccessPolicy(metaclass=ABCMeta): diff --git a/diracx-routers/src/diracx/routers/auth/__init__.py b/diracx-routers/src/diracx/routers/auth/__init__.py index 7caa17ce5..689bbb5fd 100644 --- a/diracx-routers/src/diracx/routers/auth/__init__.py +++ b/diracx-routers/src/diracx/routers/auth/__init__.py @@ -1,10 +1,13 @@ from __future__ import annotations +__all__ = ["create_token", "router", "verify_dirac_access_token"] + from ..fastapi_classes import DiracxRouter from ..utils.users import verify_dirac_access_token from .authorize_code_flow import router as authorize_code_flow_router from .device_flow import router as device_flow_router from .management import router as management_router +from .token import create_token from .token import router as token_router router = DiracxRouter(require_auth=False) @@ -12,5 +15,3 @@ router.include_router(management_router) router.include_router(authorize_code_flow_router) router.include_router(token_router) - -__all__ = ["verify_dirac_access_token"] diff --git a/diracx-routers/src/diracx/routers/auth/authorize_code_flow.py b/diracx-routers/src/diracx/routers/auth/authorize_code_flow.py index 437e63bad..d7da44fd3 100644 --- a/diracx-routers/src/diracx/routers/auth/authorize_code_flow.py +++ b/diracx-routers/src/diracx/routers/auth/authorize_code_flow.py @@ -18,10 +18,10 @@ from diracx.core.exceptions import AuthorizationError, IAMClientError, IAMServerError from diracx.core.settings import AuthSettings from diracx.db.sql import AuthDB -from diracx.logic.auth.authorize_code_flow import ( +from diracx.logic.auth import ( complete_authorization_flow as complete_authorization_flow_bl, ) -from diracx.logic.auth.authorize_code_flow import ( +from diracx.logic.auth import ( initiate_authorization_flow as initiate_authorization_flow_bl, ) from diracx.routers.dependencies import Config diff --git a/diracx-routers/src/diracx/routers/auth/device_flow.py b/diracx-routers/src/diracx/routers/auth/device_flow.py index 1d33158eb..53c3606e7 100644 --- a/diracx-routers/src/diracx/routers/auth/device_flow.py +++ b/diracx-routers/src/diracx/routers/auth/device_flow.py @@ -18,14 +18,16 @@ from sqlalchemy.exc import NoResultFound from diracx.core.exceptions import IAMClientError, IAMServerError -from diracx.core.models.auth import InitiateDeviceFlowResponse +from diracx.core.models import InitiateDeviceFlowResponse from diracx.core.settings import AuthSettings from diracx.db.sql import AuthDB -from diracx.logic.auth.device_flow import do_device_flow as do_device_flow_bl -from diracx.logic.auth.device_flow import ( +from diracx.logic.auth import ( + do_device_flow as do_device_flow_bl, +) +from diracx.logic.auth import ( finish_device_flow as finish_device_flow_bl, ) -from diracx.logic.auth.device_flow import ( +from diracx.logic.auth import ( initiate_device_flow as initiate_device_flow_bl, ) from diracx.routers.dependencies import Config diff --git a/diracx-routers/src/diracx/routers/auth/management.py b/diracx-routers/src/diracx/routers/auth/management.py index eef90d324..a054ff72b 100644 --- a/diracx-routers/src/diracx/routers/auth/management.py +++ b/diracx-routers/src/diracx/routers/auth/management.py @@ -18,18 +18,18 @@ from diracx.core.properties import PROXY_MANAGEMENT, SecurityProperty from diracx.core.settings import AuthSettings from diracx.db.sql import AuthDB -from diracx.logic.auth.management import ( +from diracx.logic.auth import ( get_refresh_tokens as get_refresh_tokens_bl, ) -from diracx.logic.auth.management import ( +from diracx.logic.auth import ( revoke_refresh_token_by_jti as revoke_refresh_token_by_jti_bl, ) -from diracx.logic.auth.management import ( +from diracx.logic.auth import ( revoke_refresh_token_by_refresh_token as revoke_refresh_token_by_refresh_token_bl, ) from ..fastapi_classes import DiracxRouter -from ..utils.users import AuthorizedUserInfo, verify_dirac_access_token +from ..utils import AuthorizedUserInfo, verify_dirac_access_token router = DiracxRouter(require_auth=False) diff --git a/diracx-routers/src/diracx/routers/auth/token.py b/diracx-routers/src/diracx/routers/auth/token.py index 78bc903d7..55e17bece 100644 --- a/diracx-routers/src/diracx/routers/auth/token.py +++ b/diracx-routers/src/diracx/routers/auth/token.py @@ -14,7 +14,7 @@ InvalidCredentialsError, PendingAuthorizationError, ) -from diracx.core.models.auth import ( +from diracx.core.models import ( AccessTokenPayload, GrantType, RefreshTokenPayload, @@ -22,9 +22,9 @@ ) from diracx.core.settings import AuthSettings from diracx.db.sql import AuthDB -from diracx.logic.auth.token import create_token -from diracx.logic.auth.token import get_oidc_token as get_oidc_token_bl -from diracx.logic.auth.token import ( +from diracx.logic.auth import create_token +from diracx.logic.auth import get_oidc_token as get_oidc_token_bl +from diracx.logic.auth import ( perform_legacy_exchange as perform_legacy_exchange_bl, ) from diracx.routers.access_policies import BaseAccessPolicy diff --git a/diracx-routers/src/diracx/routers/auth/well_known.py b/diracx-routers/src/diracx/routers/auth/well_known.py index 1690969d8..a621a34be 100644 --- a/diracx-routers/src/diracx/routers/auth/well_known.py +++ b/diracx-routers/src/diracx/routers/auth/well_known.py @@ -2,15 +2,15 @@ from fastapi import Request -from diracx.core.models.auth import Metadata, OpenIDConfiguration +from diracx.core.models import Metadata, OpenIDConfiguration from diracx.core.settings import AuthSettings -from diracx.logic.auth.well_known import ( +from diracx.logic.auth import ( get_installation_metadata as get_installation_metadata_bl, ) -from diracx.logic.auth.well_known import ( +from diracx.logic.auth import ( get_jwks as get_jwks_bl, ) -from diracx.logic.auth.well_known import ( +from diracx.logic.auth import ( get_openid_configuration as get_openid_configuration_bl, ) from diracx.routers.dependencies import Config diff --git a/diracx-routers/src/diracx/routers/configuration.py b/diracx-routers/src/diracx/routers/configuration.py index 17f4ab70b..7535f1c0d 100644 --- a/diracx-routers/src/diracx/routers/configuration.py +++ b/diracx-routers/src/diracx/routers/configuration.py @@ -1,5 +1,7 @@ from __future__ import annotations +__all__ = [] + import logging from datetime import datetime, timezone from typing import Annotated diff --git a/diracx-routers/src/diracx/routers/factory.py b/diracx-routers/src/diracx/routers/factory.py index 1700a2352..342e6d5cd 100644 --- a/diracx-routers/src/diracx/routers/factory.py +++ b/diracx-routers/src/diracx/routers/factory.py @@ -2,6 +2,8 @@ from __future__ import annotations +__all__ = ["DIRACX_MIN_CLIENT_VERSION", "create_app", "create_app_inner"] + import inspect import logging import os diff --git a/diracx-routers/src/diracx/routers/fastapi_classes.py b/diracx-routers/src/diracx/routers/fastapi_classes.py index b7875e775..7999288af 100644 --- a/diracx-routers/src/diracx/routers/fastapi_classes.py +++ b/diracx-routers/src/diracx/routers/fastapi_classes.py @@ -1,5 +1,7 @@ from __future__ import annotations +__all__ = ["DiracxRouter"] + import asyncio import contextlib from typing import Any, Callable, TypeVar, cast diff --git a/diracx-routers/src/diracx/routers/health/probes.py b/diracx-routers/src/diracx/routers/health/probes.py index 1596cf151..0afc99a7f 100644 --- a/diracx-routers/src/diracx/routers/health/probes.py +++ b/diracx-routers/src/diracx/routers/health/probes.py @@ -2,8 +2,6 @@ from __future__ import annotations -__all__ = ["router"] - import logging from fastapi import HTTPException diff --git a/diracx-routers/src/diracx/routers/jobs/__init__.py b/diracx-routers/src/diracx/routers/jobs/__init__.py index 8dd828748..5c3575aeb 100644 --- a/diracx-routers/src/diracx/routers/jobs/__init__.py +++ b/diracx-routers/src/diracx/routers/jobs/__init__.py @@ -1,10 +1,24 @@ from __future__ import annotations +__all__ = [ + "EXAMPLE_HEARTBEAT", + "EXAMPLE_METADATA", + "EXAMPLE_STATUS_UPDATES", + "EXAMPLE_SUMMARY", + "ActionType", + "SandboxAccessPolicy", + "WMSAccessPolicy", + "router", +] + import logging from ..fastapi_classes import DiracxRouter +from .access_policies import ActionType, SandboxAccessPolicy, WMSAccessPolicy +from .query import EXAMPLE_SUMMARY from .query import router as query_router from .sandboxes import router as sandboxes_router +from .status import EXAMPLE_HEARTBEAT, EXAMPLE_METADATA, EXAMPLE_STATUS_UPDATES from .status import router as status_router from .submission import router as submission_router diff --git a/diracx-routers/src/diracx/routers/jobs/access_policies.py b/diracx-routers/src/diracx/routers/jobs/access_policies.py index b0048c777..bd0b29101 100644 --- a/diracx-routers/src/diracx/routers/jobs/access_policies.py +++ b/diracx-routers/src/diracx/routers/jobs/access_policies.py @@ -6,11 +6,11 @@ from fastapi import Depends, HTTPException, status -from diracx.core.models.search import VectorSearchOperator +from diracx.core.models import VectorSearchOperator from diracx.core.properties import GENERIC_PILOT, JOB_ADMINISTRATOR, NORMAL_USER from diracx.db.sql import JobDB, SandboxMetadataDB from diracx.routers.access_policies import BaseAccessPolicy -from diracx.routers.utils.users import AuthorizedUserInfo +from diracx.routers.utils import AuthorizedUserInfo class ActionType(StrEnum): diff --git a/diracx-routers/src/diracx/routers/jobs/legacy.py b/diracx-routers/src/diracx/routers/jobs/legacy.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/diracx-routers/src/diracx/routers/jobs/query.py b/diracx-routers/src/diracx/routers/jobs/query.py index 487334601..2fdf0b1cc 100644 --- a/diracx-routers/src/diracx/routers/jobs/query.py +++ b/diracx-routers/src/diracx/routers/jobs/query.py @@ -5,20 +5,20 @@ from fastapi import Body, Depends, Query, Response -from diracx.core.models.search import ( +from diracx.core.models import ( SearchParams, SummaryParams, ) from diracx.core.properties import JOB_ADMINISTRATOR from diracx.db.os import JobParametersDB from diracx.db.sql import JobDB, JobLoggingDB -from diracx.logic.jobs.query import MAX_PER_PAGE -from diracx.logic.jobs.query import search as search_bl -from diracx.logic.jobs.query import summary as summary_bl +from diracx.logic.jobs import MAX_PER_PAGE +from diracx.logic.jobs import search as search_bl +from diracx.logic.jobs import summary as summary_bl from diracx.routers.dependencies import Config from ..fastapi_classes import DiracxRouter -from ..utils.users import AuthorizedUserInfo, verify_dirac_access_token +from ..utils import AuthorizedUserInfo, verify_dirac_access_token from .access_policies import ActionType, CheckWMSPolicyCallable router = DiracxRouter() diff --git a/diracx-routers/src/diracx/routers/jobs/sandboxes.py b/diracx-routers/src/diracx/routers/jobs/sandboxes.py index 9f1d05721..788145a9d 100644 --- a/diracx-routers/src/diracx/routers/jobs/sandboxes.py +++ b/diracx-routers/src/diracx/routers/jobs/sandboxes.py @@ -6,29 +6,29 @@ from fastapi import Body, Depends, HTTPException, Query from diracx.core.exceptions import SandboxAlreadyAssignedError, SandboxNotFoundError -from diracx.core.models.sandbox import ( +from diracx.core.models import ( SandboxDownloadResponse, SandboxInfo, SandboxUploadResponse, ) from diracx.core.settings import SandboxStoreSettings from diracx.db.sql import JobDB, SandboxMetadataDB -from diracx.logic.jobs.sandboxes import SANDBOX_PFN_REGEX -from diracx.logic.jobs.sandboxes import ( +from diracx.logic.jobs import SANDBOX_PFN_REGEX +from diracx.logic.jobs import ( assign_sandbox_to_job as assign_sandbox_to_job_bl, ) -from diracx.logic.jobs.sandboxes import get_job_sandbox as get_job_sandbox_bl -from diracx.logic.jobs.sandboxes import get_job_sandboxes as get_job_sandboxes_bl -from diracx.logic.jobs.sandboxes import get_sandbox_file as get_sandbox_file_bl -from diracx.logic.jobs.sandboxes import ( +from diracx.logic.jobs import get_job_sandbox as get_job_sandbox_bl +from diracx.logic.jobs import get_job_sandboxes as get_job_sandboxes_bl +from diracx.logic.jobs import get_sandbox_file as get_sandbox_file_bl +from diracx.logic.jobs import ( initiate_sandbox_upload as initiate_sandbox_upload_bl, ) -from diracx.logic.jobs.sandboxes import ( +from diracx.logic.jobs import ( unassign_jobs_sandboxes as unassign_jobs_sandboxes_bl, ) from ..fastapi_classes import DiracxRouter -from ..utils.users import AuthorizedUserInfo, verify_dirac_access_token +from ..utils import AuthorizedUserInfo, verify_dirac_access_token from .access_policies import ( ActionType, CheckSandboxPolicyCallable, diff --git a/diracx-routers/src/diracx/routers/jobs/status.py b/diracx-routers/src/diracx/routers/jobs/status.py index 6093d19b6..debad29bb 100644 --- a/diracx-routers/src/diracx/routers/jobs/status.py +++ b/diracx-routers/src/diracx/routers/jobs/status.py @@ -6,7 +6,7 @@ from fastapi import Body, HTTPException, Query -from diracx.core.models.job import ( +from diracx.core.models import ( HeartbeatData, JobCommand, JobMetaData, @@ -15,13 +15,13 @@ ) from diracx.db.os import JobParametersDB from diracx.db.sql import JobDB, JobLoggingDB, TaskQueueDB -from diracx.logic.jobs.status import add_heartbeat as add_heartbeat_bl -from diracx.logic.jobs.status import get_job_commands as get_job_commands_bl -from diracx.logic.jobs.status import reschedule_jobs as reschedule_jobs_bl -from diracx.logic.jobs.status import ( +from diracx.logic.jobs import add_heartbeat as add_heartbeat_bl +from diracx.logic.jobs import get_job_commands as get_job_commands_bl +from diracx.logic.jobs import reschedule_jobs as reschedule_jobs_bl +from diracx.logic.jobs import ( set_job_parameters_or_attributes as set_job_parameters_or_attributes_bl, ) -from diracx.logic.jobs.status import set_job_statuses as set_job_statuses_bl +from diracx.logic.jobs import set_job_statuses as set_job_statuses_bl from diracx.routers.dependencies import Config from ..fastapi_classes import DiracxRouter diff --git a/diracx-routers/src/diracx/routers/jobs/submission.py b/diracx-routers/src/diracx/routers/jobs/submission.py index 818f7e6e0..62f215394 100644 --- a/diracx-routers/src/diracx/routers/jobs/submission.py +++ b/diracx-routers/src/diracx/routers/jobs/submission.py @@ -6,13 +6,13 @@ from fastapi import Body, Depends, HTTPException from pydantic import BaseModel -from diracx.core.models.job import InsertedJob +from diracx.core.models import InsertedJob from diracx.db.sql import JobDB, JobLoggingDB -from diracx.logic.jobs.submission import submit_jdl_jobs as submit_jdl_jobs_bl +from diracx.logic.jobs import submit_jdl_jobs as submit_jdl_jobs_bl from diracx.routers.dependencies import Config from ..fastapi_classes import DiracxRouter -from ..utils.users import AuthorizedUserInfo, verify_dirac_access_token +from ..utils import AuthorizedUserInfo, verify_dirac_access_token from .access_policies import ActionType, CheckWMSPolicyCallable router = DiracxRouter() diff --git a/diracx-routers/src/diracx/routers/otel.py b/diracx-routers/src/diracx/routers/otel.py index be03c2f81..da7758328 100644 --- a/diracx-routers/src/diracx/routers/otel.py +++ b/diracx-routers/src/diracx/routers/otel.py @@ -1,5 +1,7 @@ from __future__ import annotations +__all__ = ["instrument_otel"] + import logging import os from typing import Optional diff --git a/diracx-routers/src/diracx/routers/utils/__init__.py b/diracx-routers/src/diracx/routers/utils/__init__.py index e69de29bb..a8c5919dc 100644 --- a/diracx-routers/src/diracx/routers/utils/__init__.py +++ b/diracx-routers/src/diracx/routers/utils/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +__all__ = ["AuthorizedUserInfo", "verify_dirac_access_token"] + +from .users import AuthorizedUserInfo, verify_dirac_access_token diff --git a/diracx-routers/src/diracx/routers/utils/users.py b/diracx-routers/src/diracx/routers/utils/users.py index fe87c5aef..8228a9ba8 100644 --- a/diracx-routers/src/diracx/routers/utils/users.py +++ b/diracx-routers/src/diracx/routers/utils/users.py @@ -13,10 +13,10 @@ from pydantic_core import CoreSchema, core_schema from uuid_utils import UUID as _UUID -from diracx.core.models.auth import UserInfo +from diracx.core.models import UserInfo from diracx.core.properties import SecurityProperty from diracx.core.settings import AuthSettings -from diracx.logic.auth.utils import read_token +from diracx.logic.auth import read_token from diracx.routers.dependencies import auto_inject logger = logging.getLogger(__name__) diff --git a/diracx-routers/tests/auth/__init__.py b/diracx-routers/tests/auth/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-routers/tests/auth/__init__.py +++ b/diracx-routers/tests/auth/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-routers/tests/auth/test_standard.py b/diracx-routers/tests/auth/test_standard.py index f55847271..0bed66b88 100644 --- a/diracx-routers/tests/auth/test_standard.py +++ b/diracx-routers/tests/auth/test_standard.py @@ -23,18 +23,18 @@ from diracx.core.config import Config from diracx.core.exceptions import AuthorizationError, IAMServerError -from diracx.core.models.auth import GrantType +from diracx.core.models import GrantType from diracx.core.properties import NORMAL_USER, PROXY_MANAGEMENT, SecurityProperty from diracx.core.settings import AuthSettings from diracx.logic.auth.token import _sign_token_payload -from diracx.logic.auth.utils import ( - _server_metadata_cache, +from diracx.logic.auth import ( decrypt_state, encrypt_state, get_server_metadata, parse_and_validate_scope, verify_dirac_refresh_token, ) +from diracx.logic.auth.utils import _server_metadata_cache DIRAC_CLIENT_ID = "myDIRACClientID" pytestmark = pytest.mark.enabled_dependencies( diff --git a/diracx-routers/tests/health/__init__.py b/diracx-routers/tests/health/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-routers/tests/health/__init__.py +++ b/diracx-routers/tests/health/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-routers/tests/jobs/__init__.py b/diracx-routers/tests/jobs/__init__.py index e69de29bb..4d21ee850 100644 --- a/diracx-routers/tests/jobs/__init__.py +++ b/diracx-routers/tests/jobs/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/diracx-routers/tests/jobs/test_query.py b/diracx-routers/tests/jobs/test_query.py index e40db9bd5..cb1097f2d 100644 --- a/diracx-routers/tests/jobs/test_query.py +++ b/diracx-routers/tests/jobs/test_query.py @@ -7,8 +7,8 @@ from fastapi.testclient import TestClient from freezegun import freeze_time -from diracx.core.models.job import JobStatus -from diracx.routers.jobs.query import EXAMPLE_SUMMARY +from diracx.core.models import JobStatus +from diracx.routers.jobs import EXAMPLE_SUMMARY from .conftest import TEST_JDL, TEST_PARAMETRIC_JDL diff --git a/diracx-routers/tests/jobs/test_status.py b/diracx-routers/tests/jobs/test_status.py index 25b69eb1f..e3ecaf3a7 100644 --- a/diracx-routers/tests/jobs/test_status.py +++ b/diracx-routers/tests/jobs/test_status.py @@ -6,8 +6,8 @@ import pytest from fastapi.testclient import TestClient -from diracx.core.models.job import JobStatus -from diracx.routers.jobs.status import ( +from diracx.core.models import JobStatus +from diracx.routers.jobs import ( EXAMPLE_HEARTBEAT, EXAMPLE_METADATA, EXAMPLE_STATUS_UPDATES, diff --git a/diracx-routers/tests/jobs/test_wms_access_policy.py b/diracx-routers/tests/jobs/test_wms_access_policy.py index 351db139f..83484b8d1 100644 --- a/diracx-routers/tests/jobs/test_wms_access_policy.py +++ b/diracx-routers/tests/jobs/test_wms_access_policy.py @@ -5,12 +5,12 @@ from uuid_utils import uuid7 from diracx.core.properties import JOB_ADMINISTRATOR, NORMAL_USER -from diracx.routers.jobs.access_policies import ( +from diracx.routers.jobs import ( ActionType, SandboxAccessPolicy, WMSAccessPolicy, ) -from diracx.routers.utils.users import AuthorizedUserInfo +from diracx.routers.utils import AuthorizedUserInfo base_payload = { "sub": "testingVO:yellow-sub", diff --git a/diracx-routers/tests/test_generic.py b/diracx-routers/tests/test_generic.py index 6a77ec8e6..a3447a226 100644 --- a/diracx-routers/tests/test_generic.py +++ b/diracx-routers/tests/test_generic.py @@ -5,7 +5,7 @@ import pytest from packaging.version import Version, parse -from diracx.routers import DIRACX_MIN_CLIENT_VERSION +from diracx.routers.factory import DIRACX_MIN_CLIENT_VERSION pytestmark = pytest.mark.enabled_dependencies( [ diff --git a/diracx-tasks/src/diracx/tasks/__init__.py b/diracx-tasks/src/diracx/tasks/__init__.py index 9d48db4f9..4d21ee850 100644 --- a/diracx-tasks/src/diracx/tasks/__init__.py +++ b/diracx-tasks/src/diracx/tasks/__init__.py @@ -1 +1,3 @@ from __future__ import annotations + +__all__ = [] diff --git a/diracx-tasks/src/diracx/tasks/depends.py b/diracx-tasks/src/diracx/tasks/depends.py index 4a070e0bb..671475820 100644 --- a/diracx-tasks/src/diracx/tasks/depends.py +++ b/diracx-tasks/src/diracx/tasks/depends.py @@ -2,6 +2,6 @@ from __future__ import annotations -from diracx.tasks.plumbing.depends import CallbackSpawner, NoTransaction +__all__ = ["CallbackSpawner", "NoTransaction"] -__all__ = ("CallbackSpawner", "NoTransaction") +from diracx.tasks.plumbing.depends import CallbackSpawner, NoTransaction diff --git a/diracx-tasks/src/diracx/tasks/jobs/__init__.py b/diracx-tasks/src/diracx/tasks/jobs/__init__.py index d97fed946..3b09465d8 100644 --- a/diracx-tasks/src/diracx/tasks/jobs/__init__.py +++ b/diracx-tasks/src/diracx/tasks/jobs/__init__.py @@ -1,5 +1,7 @@ from __future__ import annotations -from .clean_sandbox_store import CleanSandboxStoreTask +__all__ = [ + "CleanSandboxStoreTask", +] -__all__ = ("CleanSandboxStoreTask",) +from .clean_sandbox_store import CleanSandboxStoreTask diff --git a/diracx-tasks/src/diracx/tasks/plumbing/__init__.py b/diracx-tasks/src/diracx/tasks/plumbing/__init__.py index 9d48db4f9..4d21ee850 100644 --- a/diracx-tasks/src/diracx/tasks/plumbing/__init__.py +++ b/diracx-tasks/src/diracx/tasks/plumbing/__init__.py @@ -1 +1,3 @@ from __future__ import annotations + +__all__ = [] diff --git a/diracx-tasks/src/diracx/tasks/plumbing/broker/__init__.py b/diracx-tasks/src/diracx/tasks/plumbing/broker/__init__.py index 694ce0b48..560e4a856 100644 --- a/diracx-tasks/src/diracx/tasks/plumbing/broker/__init__.py +++ b/diracx-tasks/src/diracx/tasks/plumbing/broker/__init__.py @@ -11,11 +11,11 @@ from .result_backend import RedisResultBackend __all__ = [ - "RedisStreamBroker", + "ReceivedMessage", "RedisResultBackend", + "RedisStreamBroker", + "TaskBinding", "TaskMessage", "TaskResult", - "ReceivedMessage", - "TaskBinding", "submit_task", ] diff --git a/diracx-tasks/src/diracx/tasks/plumbing/broker/models.py b/diracx-tasks/src/diracx/tasks/plumbing/broker/models.py index dc4778c79..e4eb2d8ae 100644 --- a/diracx-tasks/src/diracx/tasks/plumbing/broker/models.py +++ b/diracx-tasks/src/diracx/tasks/plumbing/broker/models.py @@ -1,10 +1,10 @@ from __future__ import annotations __all__ = [ - "TaskMessage", - "TaskResult", "ReceivedMessage", "TaskBinding", + "TaskMessage", + "TaskResult", "submit_task", ] diff --git a/diracx-tasks/src/diracx/tasks/plumbing/callbacks.py b/diracx-tasks/src/diracx/tasks/plumbing/callbacks.py index b6826709b..37799325b 100644 --- a/diracx-tasks/src/diracx/tasks/plumbing/callbacks.py +++ b/diracx-tasks/src/diracx/tasks/plumbing/callbacks.py @@ -1,6 +1,6 @@ from __future__ import annotations -__all__ = ["spawn_with_callback", "fire_callback"] +__all__ = ["fire_callback", "spawn_with_callback"] import asyncio import logging diff --git a/diracx-tasks/src/diracx/tasks/plumbing/config.py b/diracx-tasks/src/diracx/tasks/plumbing/config.py index 482557089..1cbbd155d 100644 --- a/diracx-tasks/src/diracx/tasks/plumbing/config.py +++ b/diracx-tasks/src/diracx/tasks/plumbing/config.py @@ -1,6 +1,6 @@ from __future__ import annotations -__all__ = ["TasksConfig", "TaskOverride", "PeriodicTaskConfig"] +__all__ = ["PeriodicTaskConfig", "TaskOverride", "TasksConfig"] from pydantic import BaseModel diff --git a/diracx-tasks/src/diracx/tasks/plumbing/depends.py b/diracx-tasks/src/diracx/tasks/plumbing/depends.py index 4c2a6fec3..858f3afd5 100644 --- a/diracx-tasks/src/diracx/tasks/plumbing/depends.py +++ b/diracx-tasks/src/diracx/tasks/plumbing/depends.py @@ -14,14 +14,14 @@ from __future__ import annotations __all__ = ( - "Config", - "NoTransaction", - "auto_inject", - "auto_inject_depends", "AvailableSecurityProperties", "CallbackSpawner", + "Config", + "NoTransaction", "_CallbackSpawner", "_callback_spawner_placeholder", + "auto_inject", + "auto_inject_depends", ) import dataclasses diff --git a/diracx-tasks/src/diracx/tasks/plumbing/exceptions.py b/diracx-tasks/src/diracx/tasks/plumbing/exceptions.py index 9ca65e6e7..2d8a84ec4 100644 --- a/diracx-tasks/src/diracx/tasks/plumbing/exceptions.py +++ b/diracx-tasks/src/diracx/tasks/plumbing/exceptions.py @@ -1,13 +1,13 @@ from __future__ import annotations __all__ = [ - "UnableToAcquireLockError", + "ResultIsMissingError", + "SendTaskError", "StopRetryingError", + "TaskRetryRequestedError", "TooManyRetriesError", + "UnableToAcquireLockError", "UnretryableError", - "TaskRetryRequestedError", - "SendTaskError", - "ResultIsMissingError", ] diff --git a/diracx-tasks/src/diracx/tasks/plumbing/factory.py b/diracx-tasks/src/diracx/tasks/plumbing/factory.py index d9c23bf30..2dfe3f374 100644 --- a/diracx-tasks/src/diracx/tasks/plumbing/factory.py +++ b/diracx-tasks/src/diracx/tasks/plumbing/factory.py @@ -2,10 +2,10 @@ __all__ = [ "find_missing_overrides", - "task_wrapper", - "wrap_task", "load_task_registry", "setup_dependency_overrides", + "task_wrapper", + "wrap_task", ] import asyncio diff --git a/diracx-tasks/src/diracx/tasks/plumbing/locks.py b/diracx-tasks/src/diracx/tasks/plumbing/locks.py index e40b71c24..46fdcc728 100644 --- a/diracx-tasks/src/diracx/tasks/plumbing/locks.py +++ b/diracx-tasks/src/diracx/tasks/plumbing/locks.py @@ -1,13 +1,13 @@ from __future__ import annotations __all__ = [ - "BaseLock", "BaseLimiter", - "MutexLock", + "BaseLock", + "ConcurrencyLimiter", "ExclusiveRWLock", - "SharedRWLock", + "MutexLock", "RateLimiter", - "ConcurrencyLimiter", + "SharedRWLock", ] import logging diff --git a/diracx-tasks/src/diracx/tasks/plumbing/retry_policies.py b/diracx-tasks/src/diracx/tasks/plumbing/retry_policies.py index a784975ce..c2df16919 100644 --- a/diracx-tasks/src/diracx/tasks/plumbing/retry_policies.py +++ b/diracx-tasks/src/diracx/tasks/plumbing/retry_policies.py @@ -1,9 +1,9 @@ from __future__ import annotations __all__ = [ - "RetryPolicyBase", - "NoRetry", "ExponentialBackoff", + "NoRetry", + "RetryPolicyBase", ] from abc import ABC, abstractmethod diff --git a/diracx-tasks/src/diracx/tasks/plumbing/schedules.py b/diracx-tasks/src/diracx/tasks/plumbing/schedules.py index a8f5da998..7bf87d980 100644 --- a/diracx-tasks/src/diracx/tasks/plumbing/schedules.py +++ b/diracx-tasks/src/diracx/tasks/plumbing/schedules.py @@ -1,10 +1,10 @@ from __future__ import annotations __all__ = [ - "TaskScheduleBase", - "IntervalSeconds", "CronSchedule", + "IntervalSeconds", "RRuleSchedule", + "TaskScheduleBase", ] from abc import ABC, abstractmethod diff --git a/diracx-tasks/src/diracx/tasks/task_run.py b/diracx-tasks/src/diracx/tasks/task_run.py index cf832c494..a9971cfe9 100644 --- a/diracx-tasks/src/diracx/tasks/task_run.py +++ b/diracx-tasks/src/diracx/tasks/task_run.py @@ -9,6 +9,8 @@ from __future__ import annotations +__all__ = [] + import argparse import asyncio import json diff --git a/diracx-testing/src/diracx/testing/__init__.py b/diracx-testing/src/diracx/testing/__init__.py index 4e3976101..295aaa079 100644 --- a/diracx-testing/src/diracx/testing/__init__.py +++ b/diracx-testing/src/diracx/testing/__init__.py @@ -1,5 +1,27 @@ from __future__ import annotations +__all__ = [ + "ClientFactory", + "aio_moto", + "cli_env", + "client_factory", + "demo_dir", + "demo_kubectl_env", + "demo_urls", + "do_device_flow_with_dex", + "fernet_key", + "private_key", + "pytest_addoption", + "session_client_factory", + "test_auth_settings", + "test_dev_settings", + "test_login", + "test_sandbox_settings", + "verify_entry_points", + "with_cli_login", + "with_config_repo", +] + from .entrypoints import verify_entry_points from .utils import ( ClientFactory, @@ -21,25 +43,3 @@ with_cli_login, with_config_repo, ) - -__all__ = ( - "verify_entry_points", - "ClientFactory", - "do_device_flow_with_dex", - "test_login", - "pytest_addoption", - "private_key", - "fernet_key", - "test_dev_settings", - "test_auth_settings", - "aio_moto", - "test_sandbox_settings", - "session_client_factory", - "client_factory", - "with_config_repo", - "demo_dir", - "demo_urls", - "demo_kubectl_env", - "cli_env", - "with_cli_login", -) diff --git a/diracx-testing/src/diracx/testing/client_generation.py b/diracx-testing/src/diracx/testing/client_generation.py index 89b9668ca..9a3c8e1b7 100644 --- a/diracx-testing/src/diracx/testing/client_generation.py +++ b/diracx-testing/src/diracx/testing/client_generation.py @@ -1,8 +1,6 @@ from __future__ import annotations -__all__ = [ - "regenerate_client", -] +__all__ = ["regenerate_client"] import argparse import ast diff --git a/diracx-testing/src/diracx/testing/client_generation_pytest.py b/diracx-testing/src/diracx/testing/client_generation_pytest.py index 95c59c6d6..1ac930cf6 100644 --- a/diracx-testing/src/diracx/testing/client_generation_pytest.py +++ b/diracx-testing/src/diracx/testing/client_generation_pytest.py @@ -1,5 +1,7 @@ from __future__ import annotations +__all__ = [] + import pytest from diracx.testing.client_generation import regenerate_client diff --git a/diracx-testing/src/diracx/testing/dummy_osdb.py b/diracx-testing/src/diracx/testing/dummy_osdb.py index 1a627d103..10b23a7d0 100644 --- a/diracx-testing/src/diracx/testing/dummy_osdb.py +++ b/diracx-testing/src/diracx/testing/dummy_osdb.py @@ -1,5 +1,7 @@ from __future__ import annotations +__all__ = [] + import secrets from diracx.db.os.utils import BaseOSDB diff --git a/diracx-testing/src/diracx/testing/entrypoints.py b/diracx-testing/src/diracx/testing/entrypoints.py index 9a789d3a9..913868cd5 100644 --- a/diracx-testing/src/diracx/testing/entrypoints.py +++ b/diracx-testing/src/diracx/testing/entrypoints.py @@ -1,5 +1,7 @@ from __future__ import annotations +__all__ = ["verify_entry_points"] + import tomllib from importlib.metadata import PackageNotFoundError, distribution, entry_points diff --git a/diracx-testing/src/diracx/testing/mock_osdb.py b/diracx-testing/src/diracx/testing/mock_osdb.py index 98350a8b3..7cc20551a 100644 --- a/diracx-testing/src/diracx/testing/mock_osdb.py +++ b/diracx-testing/src/diracx/testing/mock_osdb.py @@ -1,9 +1,9 @@ from __future__ import annotations -__all__ = ( +__all__ = [ "MockOSDBMixin", "fake_available_osdb_implementations", -) +] import contextlib from datetime import datetime, timezone @@ -13,7 +13,7 @@ from sqlalchemy import select from sqlalchemy.dialects.sqlite import insert as sqlite_insert -from diracx.core.models.search import SearchSpec, SortSpec +from diracx.core.models import SearchSpec, SortSpec from diracx.db.sql import utils as sql_utils diff --git a/diracx-testing/src/diracx/testing/osdb.py b/diracx-testing/src/diracx/testing/osdb.py index 2e7d79851..95f052a5d 100644 --- a/diracx-testing/src/diracx/testing/osdb.py +++ b/diracx-testing/src/diracx/testing/osdb.py @@ -1,5 +1,13 @@ from __future__ import annotations +__all__ = [ + "OPENSEARCH_PORT", + "DummyOSDB", + "DummyOSDB", + "DummyOSDB", + "require_port_availability", +] + import socket from subprocess import PIPE, Popen, check_output diff --git a/diracx-testing/src/diracx/testing/routers.py b/diracx-testing/src/diracx/testing/routers.py index e96523110..83c09bc5d 100644 --- a/diracx-testing/src/diracx/testing/routers.py +++ b/diracx-testing/src/diracx/testing/routers.py @@ -1,5 +1,7 @@ from __future__ import annotations +__all__ = [] + from contextlib import asynccontextmanager from functools import partial @@ -23,7 +25,7 @@ def create_app(): * replaces the parameter DBs with sqlite-backed versions """ from diracx.db.os.utils import BaseOSDB - from diracx.routers import create_app + from diracx.routers.factory import create_app BaseOSDB.available_implementations = partial( fake_available_osdb_implementations, diff --git a/diracx-testing/src/diracx/testing/time.py b/diracx-testing/src/diracx/testing/time.py index 349709a3a..9a1b51fbe 100644 --- a/diracx-testing/src/diracx/testing/time.py +++ b/diracx-testing/src/diracx/testing/time.py @@ -20,8 +20,8 @@ from __future__ import annotations __all__ = [ - "mock_sqlite_time", "julian_date", + "mock_sqlite_time", ] import re diff --git a/diracx-testing/src/diracx/testing/utils.py b/diracx-testing/src/diracx/testing/utils.py index 3a69e1c09..d8ebb076e 100644 --- a/diracx-testing/src/diracx/testing/utils.py +++ b/diracx-testing/src/diracx/testing/utils.py @@ -2,6 +2,27 @@ from __future__ import annotations +__all__ = [ + "ClientFactory", + "aio_moto", + "cli_env", + "client_factory", + "demo_dir", + "demo_kubectl_env", + "demo_urls", + "do_device_flow_with_dex", + "fernet_key", + "private_key", + "pytest_addoption", + "session_client_factory", + "test_auth_settings", + "test_dev_settings", + "test_login", + "test_sandbox_settings", + "with_cli_login", + "with_config_repo", +] + # TODO: this needs a lot of documentation, in particular what will matter for users # are the enabled_dependencies markers import asyncio @@ -24,7 +45,7 @@ from uuid_utils import uuid7 from diracx.core.extensions import DiracEntryPoint -from diracx.core.models.auth import AccessTokenPayload, RefreshTokenPayload +from diracx.core.models import AccessTokenPayload, RefreshTokenPayload if TYPE_CHECKING: from diracx.core.settings import ( @@ -32,7 +53,7 @@ DevelopmentSettings, SandboxStoreSettings, ) - from diracx.routers.utils.users import AuthorizedUserInfo + from diracx.routers.utils import AuthorizedUserInfo # to get a string like this run: @@ -152,8 +173,8 @@ def __init__( from diracx.core.settings import ServiceSettingsBase from diracx.db.os.utils import BaseOSDB from diracx.db.sql.utils import BaseSQLDB - from diracx.routers import create_app_inner from diracx.routers.access_policies import BaseAccessPolicy + from diracx.routers.factory import create_app_inner from .mock_osdb import fake_available_osdb_implementations diff --git a/docs/dev/reference/coding-conventions.md b/docs/dev/reference/coding-conventions.md index fdfd533c0..0d8b93613 100644 --- a/docs/dev/reference/coding-conventions.md +++ b/docs/dev/reference/coding-conventions.md @@ -120,9 +120,11 @@ class Owners(Base): The following structures principles may also be followed: -- `__init__.py` should not contain code, but `__all__` -- At a package level (router for example) we have one file per system (configuration.py for example) -- If we need more files (think of jobs, which have the sandbox, the joblogging, etc), we put them in a sub module (e.g routers.job). The code goes in a specific file (job.py, joblogging.py) but we use the the __init__.py to expose the specific file +- `__init__.py` should not contain code, but `__all__` as a list +- At a package level (routers for example) we have as few files as possible +- The package level `__init__.py` should contain an empty `__all__` and the modules at that level should expose their respective public API +- If the files have much in common (think of jobs, which have the sandbox, the joblogging, etc), we put them in a subpackage (e.g routers.job). The code goes in a specific file (job.py, joblogging.py) +- At subpackage level, `__init__.py` should expose the public API through the `__all__` list by importing from the modules at that level. These submodules should not contain an `__all__` See [this issue](https://github.com/DIRACGrid/diracx/issues/268). diff --git a/extensions/gubbins/gubbins-api/src/gubbins/api/__init__.py b/extensions/gubbins/gubbins-api/src/gubbins/api/__init__.py index e69de29bb..4d21ee850 100644 --- a/extensions/gubbins/gubbins-api/src/gubbins/api/__init__.py +++ b/extensions/gubbins/gubbins-api/src/gubbins/api/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/extensions/gubbins/gubbins-cli/src/gubbins/cli/__init__.py b/extensions/gubbins/gubbins-cli/src/gubbins/cli/__init__.py index e69de29bb..4d21ee850 100644 --- a/extensions/gubbins/gubbins-cli/src/gubbins/cli/__init__.py +++ b/extensions/gubbins/gubbins-cli/src/gubbins/cli/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/extensions/gubbins/gubbins-cli/src/gubbins/cli/config.py b/extensions/gubbins/gubbins-cli/src/gubbins/cli/config.py index 5f6b81380..cb40af64a 100644 --- a/extensions/gubbins/gubbins-cli/src/gubbins/cli/config.py +++ b/extensions/gubbins/gubbins-cli/src/gubbins/cli/config.py @@ -3,6 +3,10 @@ """ # In order to extend it, just import the app from DiracX +from __future__ import annotations + +__all__ = ["dump"] + from diracx.cli.config import app diff --git a/extensions/gubbins/gubbins-cli/src/gubbins/cli/lollygag.py b/extensions/gubbins/gubbins-cli/src/gubbins/cli/lollygag.py index cd610c990..846de24be 100644 --- a/extensions/gubbins/gubbins-cli/src/gubbins/cli/lollygag.py +++ b/extensions/gubbins/gubbins-cli/src/gubbins/cli/lollygag.py @@ -2,6 +2,10 @@ This shows how to create a new subcommand """ +from __future__ import annotations + +__all__ = ["app"] + from diracx.cli.utils import AsyncTyper from gubbins.client.aio import AsyncGubbinsClient diff --git a/extensions/gubbins/gubbins-client/src/gubbins/client/__init__.py b/extensions/gubbins/gubbins-client/src/gubbins/client/__init__.py index 0f51f489d..5470c74bd 100644 --- a/extensions/gubbins/gubbins-client/src/gubbins/client/__init__.py +++ b/extensions/gubbins/gubbins-client/src/gubbins/client/__init__.py @@ -1,9 +1,3 @@ from __future__ import absolute_import -__all__ = [ - "aio", - "models", - "sync", -] - -from . import aio, models, sync +__all__ = [] diff --git a/extensions/gubbins/gubbins-core/src/gubbins/core/__init__.py b/extensions/gubbins/gubbins-core/src/gubbins/core/__init__.py index 28d47ffbd..4d21ee850 100644 --- a/extensions/gubbins/gubbins-core/src/gubbins/core/__init__.py +++ b/extensions/gubbins/gubbins-core/src/gubbins/core/__init__.py @@ -1 +1,3 @@ -__all__ = ("config", "properties", "models") +from __future__ import annotations + +__all__ = [] diff --git a/extensions/gubbins/gubbins-core/src/gubbins/core/config/__init__.py b/extensions/gubbins/gubbins-core/src/gubbins/core/config/__init__.py index 1d6a90374..c4a9924eb 100644 --- a/extensions/gubbins/gubbins-core/src/gubbins/core/config/__init__.py +++ b/extensions/gubbins/gubbins-core/src/gubbins/core/config/__init__.py @@ -1 +1,5 @@ -__all__ = ("schema",) +from __future__ import annotations + +__all__ = ["Config"] + +from .schema import Config diff --git a/extensions/gubbins/gubbins-core/src/gubbins/core/config/schema.py b/extensions/gubbins/gubbins-core/src/gubbins/core/config/schema.py index ccc383966..3f266488d 100644 --- a/extensions/gubbins/gubbins-core/src/gubbins/core/config/schema.py +++ b/extensions/gubbins/gubbins-core/src/gubbins/core/config/schema.py @@ -2,13 +2,13 @@ from typing import MutableMapping -from diracx.core.config.schema import ( +from diracx.core.config import ( Config as _Config, ) -from diracx.core.config.schema import ( +from diracx.core.config import ( RegistryConfig as _RegistryConfig, ) -from diracx.core.config.schema import ( +from diracx.core.config import ( UserConfig as _UserConfig, ) diff --git a/extensions/gubbins/gubbins-core/src/gubbins/core/models.py b/extensions/gubbins/gubbins-core/src/gubbins/core/models.py index 9497b667b..9d2e40c3f 100644 --- a/extensions/gubbins/gubbins-core/src/gubbins/core/models.py +++ b/extensions/gubbins/gubbins-core/src/gubbins/core/models.py @@ -1,4 +1,8 @@ -from diracx.core.models.auth import Metadata +from __future__ import annotations + +from diracx.core.models import Metadata + +__all__ = ["ExtendedMetadata"] class ExtendedMetadata(Metadata): diff --git a/extensions/gubbins/gubbins-core/src/gubbins/core/properties.py b/extensions/gubbins/gubbins-core/src/gubbins/core/properties.py index a14a23376..9dc1a5a04 100644 --- a/extensions/gubbins/gubbins-core/src/gubbins/core/properties.py +++ b/extensions/gubbins/gubbins-core/src/gubbins/core/properties.py @@ -1,5 +1,7 @@ from __future__ import annotations +__all__ = ["GUBBINS_SENSEI"] + from diracx.core.properties import SecurityProperty GUBBINS_SENSEI = SecurityProperty("GubbinsSensei") diff --git a/extensions/gubbins/gubbins-core/tests/test_config.py b/extensions/gubbins/gubbins-core/tests/test_config.py index 13a137334..9833decb3 100644 --- a/extensions/gubbins/gubbins-core/tests/test_config.py +++ b/extensions/gubbins/gubbins-core/tests/test_config.py @@ -8,7 +8,7 @@ import pytest from diracx.core.config import ConfigSource, RemoteGitConfigSource -from gubbins.core.config.schema import Config +from gubbins.core.config import Config # The diracx-chart contains a CS example TEST_REPO = "git+https://github.com/DIRACGrid/diracx-charts/" diff --git a/extensions/gubbins/gubbins-db/src/gubbins/db/__init__.py b/extensions/gubbins/gubbins-db/src/gubbins/db/__init__.py index 7a32a33be..f6ea1ad0c 100644 --- a/extensions/gubbins/gubbins-db/src/gubbins/db/__init__.py +++ b/extensions/gubbins/gubbins-db/src/gubbins/db/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations # Do not forget that, as otherwise diracx won't find your DBs -__all__ = ("sql",) +__all__ = ["sql"] from . import sql diff --git a/extensions/gubbins/gubbins-db/src/gubbins/db/sql/__init__.py b/extensions/gubbins/gubbins-db/src/gubbins/db/sql/__init__.py index 4948ac385..7483ecf78 100644 --- a/extensions/gubbins/gubbins-db/src/gubbins/db/sql/__init__.py +++ b/extensions/gubbins/gubbins-db/src/gubbins/db/sql/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations -__all__ = ("LollygagDB", "GubbinsJobDB") +__all__ = ["LollygagDB", "GubbinsJobDB"] from .jobs.db import GubbinsJobDB from .lollygag.db import LollygagDB diff --git a/extensions/gubbins/gubbins-db/src/gubbins/db/sql/jobs/__init__.py b/extensions/gubbins/gubbins-db/src/gubbins/db/sql/jobs/__init__.py index e69de29bb..4d21ee850 100644 --- a/extensions/gubbins/gubbins-db/src/gubbins/db/sql/jobs/__init__.py +++ b/extensions/gubbins/gubbins-db/src/gubbins/db/sql/jobs/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/extensions/gubbins/gubbins-db/src/gubbins/db/sql/jobs/db.py b/extensions/gubbins/gubbins-db/src/gubbins/db/sql/jobs/db.py index d5e86c333..bb15bcf94 100644 --- a/extensions/gubbins/gubbins-db/src/gubbins/db/sql/jobs/db.py +++ b/extensions/gubbins/gubbins-db/src/gubbins/db/sql/jobs/db.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from diracx.db.sql.job.db import JobDB from sqlalchemy import insert, select diff --git a/extensions/gubbins/gubbins-db/src/gubbins/db/sql/jobs/schema.py b/extensions/gubbins/gubbins-db/src/gubbins/db/sql/jobs/schema.py index ceb2675c5..1a6231c8b 100644 --- a/extensions/gubbins/gubbins-db/src/gubbins/db/sql/jobs/schema.py +++ b/extensions/gubbins/gubbins-db/src/gubbins/db/sql/jobs/schema.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from diracx.db.sql.job.db import JobDBBase from diracx.db.sql.job.schema import str255 from sqlalchemy import ForeignKey diff --git a/extensions/gubbins/gubbins-db/src/gubbins/db/sql/lollygag/__init__.py b/extensions/gubbins/gubbins-db/src/gubbins/db/sql/lollygag/__init__.py index e69de29bb..4d21ee850 100644 --- a/extensions/gubbins/gubbins-db/src/gubbins/db/sql/lollygag/__init__.py +++ b/extensions/gubbins/gubbins-db/src/gubbins/db/sql/lollygag/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/extensions/gubbins/gubbins-db/src/gubbins/db/sql/lollygag/schema.py b/extensions/gubbins/gubbins-db/src/gubbins/db/sql/lollygag/schema.py index 223378a4d..1d029a4c0 100644 --- a/extensions/gubbins/gubbins-db/src/gubbins/db/sql/lollygag/schema.py +++ b/extensions/gubbins/gubbins-db/src/gubbins/db/sql/lollygag/schema.py @@ -1,5 +1,7 @@ # The utils class define some boilerplate types that should be used # in place of the SQLAlchemy one. Have a look at them +from __future__ import annotations + from uuid import UUID from diracx.db.sql.utils import datetime_now, str255 diff --git a/extensions/gubbins/gubbins-db/src/gubbins/db/sql/my_pilot_db/__init__.py b/extensions/gubbins/gubbins-db/src/gubbins/db/sql/my_pilot_db/__init__.py index e69de29bb..4d21ee850 100644 --- a/extensions/gubbins/gubbins-db/src/gubbins/db/sql/my_pilot_db/__init__.py +++ b/extensions/gubbins/gubbins-db/src/gubbins/db/sql/my_pilot_db/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/extensions/gubbins/gubbins-logic/src/gubbins/logic/__init__.py b/extensions/gubbins/gubbins-logic/src/gubbins/logic/__init__.py index e69de29bb..4d21ee850 100644 --- a/extensions/gubbins/gubbins-logic/src/gubbins/logic/__init__.py +++ b/extensions/gubbins/gubbins-logic/src/gubbins/logic/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +__all__ = [] diff --git a/extensions/gubbins/gubbins-logic/src/gubbins/logic/auth/__init__.py b/extensions/gubbins/gubbins-logic/src/gubbins/logic/auth/__init__.py index e69de29bb..ccdd4074e 100644 --- a/extensions/gubbins/gubbins-logic/src/gubbins/logic/auth/__init__.py +++ b/extensions/gubbins/gubbins-logic/src/gubbins/logic/auth/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +__all__ = ["get_installation_metadata"] + +from .well_known import get_installation_metadata diff --git a/extensions/gubbins/gubbins-logic/src/gubbins/logic/auth/well_known.py b/extensions/gubbins/gubbins-logic/src/gubbins/logic/auth/well_known.py index 65c7c3f85..73a192acd 100644 --- a/extensions/gubbins/gubbins-logic/src/gubbins/logic/auth/well_known.py +++ b/extensions/gubbins/gubbins-logic/src/gubbins/logic/auth/well_known.py @@ -1,8 +1,8 @@ -from diracx.logic.auth.well_known import ( +from diracx.logic.auth import ( get_installation_metadata as get_general_installation_metadata, ) -from gubbins.core.config.schema import Config +from gubbins.core.config import Config from gubbins.core.models import ExtendedMetadata diff --git a/extensions/gubbins/gubbins-logic/src/gubbins/logic/lollygag/__init__.py b/extensions/gubbins/gubbins-logic/src/gubbins/logic/lollygag/__init__.py index e69de29bb..960920931 100644 --- a/extensions/gubbins/gubbins-logic/src/gubbins/logic/lollygag/__init__.py +++ b/extensions/gubbins/gubbins-logic/src/gubbins/logic/lollygag/__init__.py @@ -0,0 +1,5 @@ +from __future__ import annotations + +__all__ = ["get_gubbins_secrets", "get_owner_object", "insert_owner_object"] + +from .lollygag import get_gubbins_secrets, get_owner_object, insert_owner_object diff --git a/extensions/gubbins/gubbins-logic/src/gubbins/logic/my_pilots.py b/extensions/gubbins/gubbins-logic/src/gubbins/logic/my_pilots.py index f29a50b31..76f12a620 100644 --- a/extensions/gubbins/gubbins-logic/src/gubbins/logic/my_pilots.py +++ b/extensions/gubbins/gubbins-logic/src/gubbins/logic/my_pilots.py @@ -1,5 +1,13 @@ from __future__ import annotations +__all__ = [ + "get_available_ces", + "get_pilot_summary", + "submit_pilot", + "transition_pilot_states", + "PilotSubmissionError", +] + import random from gubbins.db.sql.my_pilot_db.db import MyPilotDB diff --git a/extensions/gubbins/gubbins-routers/src/gubbins/routers/__init__.py b/extensions/gubbins/gubbins-routers/src/gubbins/routers/__init__.py index c6c689af7..4d21ee850 100644 --- a/extensions/gubbins/gubbins-routers/src/gubbins/routers/__init__.py +++ b/extensions/gubbins/gubbins-routers/src/gubbins/routers/__init__.py @@ -1 +1,3 @@ -__all__ = ("dependencies", "lollygag", "well_known") +from __future__ import annotations + +__all__ = [] diff --git a/extensions/gubbins/gubbins-routers/src/gubbins/routers/dependencies.py b/extensions/gubbins/gubbins-routers/src/gubbins/routers/dependencies.py index 9a27fd9ef..abb9dde04 100644 --- a/extensions/gubbins/gubbins-routers/src/gubbins/routers/dependencies.py +++ b/extensions/gubbins/gubbins-routers/src/gubbins/routers/dependencies.py @@ -1,13 +1,13 @@ from __future__ import annotations -__all__ = ("Config",) +__all__ = ["Config"] from typing import Annotated from diracx.core.config import ConfigSource from fastapi import Depends -from gubbins.core.config.schema import Config as _Config +from gubbins.core.config import Config as _Config # Overwrite the Config dependency such that gubbins routers # can use it diff --git a/extensions/gubbins/gubbins-routers/src/gubbins/routers/lollygag/__init__.py b/extensions/gubbins/gubbins-routers/src/gubbins/routers/lollygag/__init__.py index fdbeb7ea0..179a22b31 100644 --- a/extensions/gubbins/gubbins-routers/src/gubbins/routers/lollygag/__init__.py +++ b/extensions/gubbins/gubbins-routers/src/gubbins/routers/lollygag/__init__.py @@ -1,2 +1,5 @@ -__all__ = ("router",) +from __future__ import annotations + +__all__ = ["router"] + from .lollygag import router diff --git a/extensions/gubbins/gubbins-routers/src/gubbins/routers/lollygag/access_policy.py b/extensions/gubbins/gubbins-routers/src/gubbins/routers/lollygag/access_policy.py index c2bc17f1b..36b6551f5 100644 --- a/extensions/gubbins/gubbins-routers/src/gubbins/routers/lollygag/access_policy.py +++ b/extensions/gubbins/gubbins-routers/src/gubbins/routers/lollygag/access_policy.py @@ -11,7 +11,7 @@ from typing import Annotated from diracx.routers.access_policies import BaseAccessPolicy -from diracx.routers.utils.users import AuthorizedUserInfo +from diracx.routers.utils import AuthorizedUserInfo from fastapi import Depends, HTTPException, status from gubbins.core.properties import GUBBINS_SENSEI diff --git a/extensions/gubbins/gubbins-routers/src/gubbins/routers/lollygag/lollygag.py b/extensions/gubbins/gubbins-routers/src/gubbins/routers/lollygag/lollygag.py index 41dfb9251..91abba791 100644 --- a/extensions/gubbins/gubbins-routers/src/gubbins/routers/lollygag/lollygag.py +++ b/extensions/gubbins/gubbins-routers/src/gubbins/routers/lollygag/lollygag.py @@ -8,11 +8,11 @@ from diracx.routers.fastapi_classes import DiracxRouter from gubbins.db.sql import LollygagDB -from gubbins.logic.lollygag.lollygag import ( +from gubbins.logic.lollygag import ( get_gubbins_secrets as get_gubbins_secrets_bl, ) -from gubbins.logic.lollygag.lollygag import get_owner_object as get_owner_object_bl -from gubbins.logic.lollygag.lollygag import ( +from gubbins.logic.lollygag import get_owner_object as get_owner_object_bl +from gubbins.logic.lollygag import ( insert_owner_object as insert_owner_object_bl, ) diff --git a/extensions/gubbins/gubbins-routers/src/gubbins/routers/my_pilots.py b/extensions/gubbins/gubbins-routers/src/gubbins/routers/my_pilots.py index fbf439d52..cf8ba5af8 100644 --- a/extensions/gubbins/gubbins-routers/src/gubbins/routers/my_pilots.py +++ b/extensions/gubbins/gubbins-routers/src/gubbins/routers/my_pilots.py @@ -8,6 +8,8 @@ from __future__ import annotations +__all__ = [] + from collections.abc import Callable from enum import StrEnum, auto from typing import Annotated diff --git a/extensions/gubbins/gubbins-routers/src/gubbins/routers/well_known.py b/extensions/gubbins/gubbins-routers/src/gubbins/routers/well_known.py index a443c2a7a..1e33de91d 100644 --- a/extensions/gubbins/gubbins-routers/src/gubbins/routers/well_known.py +++ b/extensions/gubbins/gubbins-routers/src/gubbins/routers/well_known.py @@ -6,11 +6,15 @@ * uses the Gubbins dependencies """ +from __future__ import annotations + +__all__ = ["get_installation_metadata"] + from diracx.routers.auth.well_known import router as diracx_wellknown_router from diracx.routers.fastapi_classes import DiracxRouter from gubbins.core.models import ExtendedMetadata -from gubbins.logic.auth.well_known import ( +from gubbins.logic.auth import ( get_installation_metadata as get_installation_metadata_bl, ) from gubbins.routers.dependencies import Config diff --git a/extensions/gubbins/gubbins-routers/tests/test_gubbins_job_manager.py b/extensions/gubbins/gubbins-routers/tests/test_gubbins_job_manager.py index b26b5482f..11caaa145 100644 --- a/extensions/gubbins/gubbins-routers/tests/test_gubbins_job_manager.py +++ b/extensions/gubbins/gubbins-routers/tests/test_gubbins_job_manager.py @@ -5,7 +5,7 @@ from datetime import datetime, timezone import pytest -from diracx.core.models.job import JobStatus +from diracx.core.models import JobStatus from fastapi.testclient import TestClient pytestmark = pytest.mark.enabled_dependencies( diff --git a/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/__init__.py b/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/__init__.py index 9d48db4f9..4d21ee850 100644 --- a/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/__init__.py +++ b/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/__init__.py @@ -1 +1,3 @@ from __future__ import annotations + +__all__ = [] diff --git a/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/depends.py b/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/depends.py index b79688aef..5a25aa04d 100644 --- a/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/depends.py +++ b/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/depends.py @@ -5,3 +5,5 @@ """ from __future__ import annotations + +__all__ = [] diff --git a/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/lock_types.py b/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/lock_types.py index 441c67586..5775afab1 100644 --- a/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/lock_types.py +++ b/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/lock_types.py @@ -6,6 +6,8 @@ from __future__ import annotations +__all__ = ["LOLLYGAG"] + from diracx.tasks.plumbing.lock_registry import register_locked_object_type LOLLYGAG = register_locked_object_type("lollygag") diff --git a/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/lollygag.py b/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/lollygag.py index 3126b7ba0..b306e3aa1 100644 --- a/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/lollygag.py +++ b/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/lollygag.py @@ -9,6 +9,8 @@ from __future__ import annotations +__all__ = ["OwnerCleanupTask", "OwnerReportTask", "SyncOwnersTask"] + import dataclasses import logging from typing import Any @@ -25,7 +27,7 @@ from diracx.tasks.plumbing.schedules import CronSchedule, IntervalSeconds from gubbins.db.sql import LollygagDB -from gubbins.logic.lollygag.lollygag import get_owner_object, insert_owner_object +from gubbins.logic.lollygag import get_owner_object, insert_owner_object from .lock_types import LOLLYGAG diff --git a/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/my_pilot_lock_types.py b/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/my_pilot_lock_types.py index cd11bd796..5a80cf880 100644 --- a/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/my_pilot_lock_types.py +++ b/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/my_pilot_lock_types.py @@ -6,6 +6,8 @@ from __future__ import annotations +__all__ = ["MY_PILOT"] + from diracx.tasks.plumbing.lock_registry import register_locked_object_type MY_PILOT = register_locked_object_type("my_pilot") diff --git a/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/my_pilots.py b/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/my_pilots.py index 1005519b0..adf9e1ba4 100644 --- a/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/my_pilots.py +++ b/extensions/gubbins/gubbins-tasks/src/gubbins/tasks/my_pilots.py @@ -12,6 +12,13 @@ # --8<-- [start:my_pilot_task_imports] from __future__ import annotations +__all__ = [ + "MyCheckPilotsTask", + "MyPilotReportTask", + "MyPilotTask", + "MySubmitPilotsTask", +] + import dataclasses import logging from typing import Any diff --git a/extensions/gubbins/gubbins-testing/src/gubbins/testing/__init__.py b/extensions/gubbins/gubbins-testing/src/gubbins/testing/__init__.py index 9d48db4f9..4d21ee850 100644 --- a/extensions/gubbins/gubbins-testing/src/gubbins/testing/__init__.py +++ b/extensions/gubbins/gubbins-testing/src/gubbins/testing/__init__.py @@ -1 +1,3 @@ from __future__ import annotations + +__all__ = [] diff --git a/pyproject.toml b/pyproject.toml index 3c970fdc1..a65d34e07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ select = [ "S", # flake8-bandit "N", # pep8-naming "INP", # flake8-no-implicit-namespace-packages + "RUF022", ] ignore = [ "B905", diff --git a/scripts/check_init_files_precommit_hook.py b/scripts/check_init_files_precommit_hook.py new file mode 100644 index 000000000..cb0d240de --- /dev/null +++ b/scripts/check_init_files_precommit_hook.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +"""Pre-commit hook for checking __all__ dunder in __init__.py files.""" + +from __future__ import annotations + +import ast +import sys + + +def main(): + """Check if the __all__ dunder exists and it's a list.""" + files_without_all = [] + files_not_list_all = [] + + for file in sys.argv[1:]: + with open(file, "r", encoding="utf-8") as f: + content = f.read() + + module_tree = ast.parse(content) + found = False + + # Go through the module + for node in module_tree.body: + # __all__ = {something} or __all__ += {something} + if isinstance(node, ast.Assign) or isinstance(node, ast.AugAssign): + if isinstance(node, ast.Assign): + target = node.targets[0] + + elif isinstance(node, ast.AugAssign): + target = node.target + + # The target is the __all__ dunder + if isinstance(target, ast.Name) and target.id == "__all__": + found = True + # __all__ is a list + if not isinstance(node.value, ast.List): + files_not_list_all.append(file) + + break + + # from {something} import __all__ + elif isinstance(node, ast.ImportFrom): + target = node.names[0] + + # The target is the __all__ dunder + if isinstance(target, ast.alias) and target.name == "__all__": + found = True + break + + if not found: + files_without_all.append(file) + + if not (files_without_all or files_not_list_all): + return 0 + + for file in files_without_all: + print(f"- {file}: __all__ not found") + + for file in files_not_list_all: + print(f"- {file}: __all__ is not a list") + + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/generate_pixi_tasks_doc.py b/scripts/generate_pixi_tasks_doc.py similarity index 98% rename from .github/workflows/generate_pixi_tasks_doc.py rename to scripts/generate_pixi_tasks_doc.py index 85c6f4cd7..c8ec5df9d 100644 --- a/.github/workflows/generate_pixi_tasks_doc.py +++ b/scripts/generate_pixi_tasks_doc.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + from __future__ import annotations import json