From 4414226bc5f24b1c8772a50b52b4320ebdce39d2 Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Tue, 21 Apr 2026 18:32:00 +0200 Subject: [PATCH 1/5] chore: drop boto for signurlarity --- diracx-core/pyproject.toml | 1 + diracx-core/src/diracx/core/s3.py | 48 ++++++++++++++++++----- diracx-core/src/diracx/core/settings.py | 20 ++++------ diracx-core/tests/test_s3.py | 12 +++--- diracx-logic/tests/jobs/test_sandboxes.py | 4 +- 5 files changed, 55 insertions(+), 30 deletions(-) diff --git a/diracx-core/pyproject.toml b/diracx-core/pyproject.toml index 720968401..a3224bec2 100644 --- a/diracx-core/pyproject.toml +++ b/diracx-core/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "pydantic-settings", "pyyaml", "sh", + "signurlarity >=0.2.2", "diraccommon >=9.0.18", ] dynamic = ["version"] diff --git a/diracx-core/src/diracx/core/s3.py b/diracx-core/src/diracx/core/s3.py index afc0c9d20..bcebf5f9f 100644 --- a/diracx-core/src/diracx/core/s3.py +++ b/diracx-core/src/diracx/core/s3.py @@ -13,14 +13,14 @@ import base64 from typing import TYPE_CHECKING, TypedDict, cast -from botocore.errorfactory import ClientError +from signurlarity.exceptions import NoSuchBucketError, PresignError from .models.sandbox import ChecksumAlgorithm if TYPE_CHECKING: from typing import TypedDict - from types_aiobotocore_s3.client import S3Client + from signurlarity.aio.client import AsyncClient class S3Object(TypedDict): Key: str @@ -31,12 +31,12 @@ class S3PresignedPostInfo(TypedDict): fields: dict[str, str] -async def s3_bucket_exists(s3_client: S3Client, bucket_name: str) -> bool: +async def s3_bucket_exists(s3_client: AsyncClient, bucket_name: str) -> bool: """Check if a bucket exists in S3.""" return await _s3_exists(s3_client.head_bucket, Bucket=bucket_name) -async def s3_object_exists(s3_client: S3Client, bucket_name: str, key: str) -> bool: +async def s3_object_exists(s3_client: AsyncClient, bucket_name: str, key: str) -> bool: """Check if an object exists in an S3 bucket.""" return await _s3_exists(s3_client.head_object, Bucket=bucket_name, Key=key) @@ -44,16 +44,46 @@ async def s3_object_exists(s3_client: S3Client, bucket_name: str, key: str) -> b async def _s3_exists(method, **kwargs: str) -> bool: try: await method(**kwargs) - except ClientError as e: - if e.response["Error"]["Code"] != "404": - raise + except (NoSuchBucketError, PresignError): + # if e.response["Error"]["Code"] != "404": + # raise return False else: return True async def generate_presigned_upload( - s3_client: S3Client, + s3_client: AsyncClient, + bucket_name: str, + key: str, + checksum_algorithm: ChecksumAlgorithm, + checksum: str, + size: int, + validity_seconds: int, +) -> S3PresignedPostInfo: + """Generate a presigned URL and fields for uploading a file to S3. + + The signature is restricted to only accept data with the given checksum and size. + """ + fields = { + "x-amz-checksum-algorithm": checksum_algorithm, + f"x-amz-checksum-{checksum_algorithm}": b16_to_b64(checksum), + } + conditions = [["content-length-range", size, size]] + [ + {k: v} for k, v in fields.items() + ] + result = await s3_client.generate_presigned_post( + Bucket=bucket_name, + Key=key, + Fields=fields, + Conditions=conditions, + ExpiresIn=validity_seconds, + ) + return cast(S3PresignedPostInfo, result) + + +async def generate_presigned_download( + s3_client: AsyncClient, bucket_name: str, key: str, checksum_algorithm: ChecksumAlgorithm, @@ -126,7 +156,7 @@ async def _s3_delete_chunk_with_retry( Bucket=bucket, Delete={"Objects": remaining, "Quiet": True}, ) - except ClientError: + except NoSuchBucketError: if attempt == max_attempts: return {obj["Key"] for obj in remaining} await asyncio.sleep(delay) diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index 40c75afd2..93a292975 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -18,9 +18,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Annotated, Any, Self, TypeVar, cast -from aiobotocore.session import get_session -from botocore.config import Config -from botocore.errorfactory import ClientError from cryptography.fernet import Fernet from joserfc.jwk import KeySet, KeySetSerialization from pydantic import ( @@ -35,9 +32,11 @@ model_validator, ) from pydantic_settings import BaseSettings, SettingsConfigDict +from signurlarity.aio.client import AsyncClient +from signurlarity.exceptions import NoSuchBucketError if TYPE_CHECKING: - from types_aiobotocore_s3.client import S3Client + from signurlarity.aio.client import AsyncClient T = TypeVar("T") @@ -324,17 +323,12 @@ class SandboxStoreSettings(ServiceSettingsBase): Controls parallelism of database DELETE operations. """ - _client: S3Client = PrivateAttr() + _client: AsyncClient = PrivateAttr() @contextlib.asynccontextmanager async def lifetime_function(self) -> AsyncIterator[None]: - async with get_session().create_client( - "s3", + async with AsyncClient( **self.s3_client_kwargs, - config=Config( - signature_version="v4", - max_pool_connections=self.s3_max_pool_connections, - ), ) as self._client: # type: ignore if not await s3_bucket_exists(self._client, self.bucket_name): if not self.auto_create_bucket: @@ -343,7 +337,7 @@ async def lifetime_function(self) -> AsyncIterator[None]: ) try: await self._client.create_bucket(Bucket=self.bucket_name) - except ClientError as e: + except NoSuchBucketError as e: raise ValueError( f"Failed to create bucket {self.bucket_name}" ) from e @@ -351,7 +345,7 @@ async def lifetime_function(self) -> AsyncIterator[None]: yield @property - def s3_client(self) -> S3Client: + def s3_client(self) -> AsyncClient: if self._client is None: raise RuntimeError("S3 client accessed before lifetime function") return self._client diff --git a/diracx-core/tests/test_s3.py b/diracx-core/tests/test_s3.py index 42f6ea022..80e2cf46d 100644 --- a/diracx-core/tests/test_s3.py +++ b/diracx-core/tests/test_s3.py @@ -7,7 +7,7 @@ import httpx import pytest -from aiobotocore.session import get_session +from signurlarity.aio.client import AsyncClient from diracx.core.s3 import ( b16_to_b64, @@ -52,7 +52,7 @@ async def moto_s3(aio_moto): Note that this is not a complete S3 backend, in particular authentication and validation of requests is not implemented. """ - async with get_session().create_client("s3", **aio_moto) as client: + async with AsyncClient(**aio_moto) as client: await client.create_bucket(Bucket=BUCKET_NAME) await client.create_bucket(Bucket=OTHER_BUCKET_NAME) yield client @@ -92,15 +92,15 @@ async def test_presigned_upload_moto(moto_s3): assert r.status_code == 204, r.text # Make sure the object is actually there - obj = await moto_s3.get_object(Bucket=BUCKET_NAME, Key=key) - assert (await obj["Body"].read()) == file_content + response = await moto_s3.list_objects(Bucket=BUCKET_NAME) + for obj in response["Contents"]: + assert obj["Key"] == key @pytest.fixture(scope="function") async def minio_client(demo_urls): """Create a S3 client that uses minio from the demo as backend.""" - async with get_session().create_client( - "s3", + async with AsyncClient( endpoint_url=demo_urls["minio"], aws_access_key_id="console", aws_secret_access_key="console123", diff --git a/diracx-logic/tests/jobs/test_sandboxes.py b/diracx-logic/tests/jobs/test_sandboxes.py index 6dc59197d..1558d6951 100644 --- a/diracx-logic/tests/jobs/test_sandboxes.py +++ b/diracx-logic/tests/jobs/test_sandboxes.py @@ -6,10 +6,10 @@ from io import BytesIO from typing import AsyncGenerator, Generator -import botocore.exceptions import freezegun import httpx import pytest +import signurlarity.exceptions import sqlalchemy from diracx.core.exceptions import SandboxNotFoundError @@ -142,7 +142,7 @@ async def test_upload_and_clean( await clean_sandboxes(sandbox_metadata_db, sandbox_settings) # Check that the sandbox was actually removed from the bucket - with pytest.raises(botocore.exceptions.ClientError, match="Not Found"): + with pytest.raises(signurlarity.exceptions.NoSuchBucketError, match="Not Found"): await sandbox_settings.s3_client.head_object( Bucket=sandbox_settings.bucket_name, Key=key ) From 27e90da2ba1ef70f21eff4eda2c54326373c42bc Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Tue, 21 Apr 2026 18:34:42 +0200 Subject: [PATCH 2/5] chore: actually remove boto dependencies --- .pre-commit-config.yaml | 2 -- diracx-core/pyproject.toml | 5 ----- extensions/gubbins/gubbins-logic/pyproject.toml | 3 --- extensions/gubbins/gubbins-routers/pyproject.toml | 3 --- 4 files changed, 13 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1fd11f86d..06401f051 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,8 +49,6 @@ repos: - types-requests - types-python-dateutil - types-croniter - - types-aiobotocore[essential] - - boto3-stubs[essential] exclude: ^(diracx-client/src/diracx/client/_generated|diracx-[a-z]+/tests/|diracx-testing/|build|extensions/gubbins/gubbins-client/src/gubbins/client/_generated) - repo: https://github.com/hukkin/mdformat diff --git a/diracx-core/pyproject.toml b/diracx-core/pyproject.toml index a3224bec2..4e31ab77e 100644 --- a/diracx-core/pyproject.toml +++ b/diracx-core/pyproject.toml @@ -13,8 +13,6 @@ classifiers = [ "Topic :: System :: Distributed Computing", ] dependencies = [ - "aiobotocore>=2.15", - "botocore>=1.35", "cachetools", "email_validator", "gitpython", @@ -34,9 +32,6 @@ testing = [ "moto[server]", ] types = [ - "botocore-stubs", - "types-aiobotocore[essential]", - "types-aiobotocore-s3", "types-cachetools", "types-PyYAML", ] diff --git a/extensions/gubbins/gubbins-logic/pyproject.toml b/extensions/gubbins/gubbins-logic/pyproject.toml index af465c4b2..4f7c9a3de 100644 --- a/extensions/gubbins/gubbins-logic/pyproject.toml +++ b/extensions/gubbins/gubbins-logic/pyproject.toml @@ -23,9 +23,6 @@ dynamic = ["version"] [project.optional-dependencies] testing = [] types = [ - "boto3-stubs", - "types-aiobotocore[essential]", - "types-aiobotocore-s3", "types-cachetools", "types-python-dateutil", "types-PyYAML", diff --git a/extensions/gubbins/gubbins-routers/pyproject.toml b/extensions/gubbins/gubbins-routers/pyproject.toml index ffc5313e4..e339729e1 100644 --- a/extensions/gubbins/gubbins-routers/pyproject.toml +++ b/extensions/gubbins/gubbins-routers/pyproject.toml @@ -23,9 +23,6 @@ dynamic = ["version"] [project.optional-dependencies] testing = ["diracx-testing", "moto[server]", "pytest-httpx"] types = [ - "boto3-stubs", - "types-aiobotocore[essential]", - "types-aiobotocore-s3", "types-cachetools", "types-python-dateutil", "types-PyYAML", From e470d635661fb103da7b687737278636f915aca0 Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Wed, 22 Apr 2026 14:44:42 +0200 Subject: [PATCH 3/5] test: get the s3 object using presigned url --- diracx-core/tests/test_s3.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/diracx-core/tests/test_s3.py b/diracx-core/tests/test_s3.py index 80e2cf46d..e0ddd3531 100644 --- a/diracx-core/tests/test_s3.py +++ b/diracx-core/tests/test_s3.py @@ -92,9 +92,17 @@ async def test_presigned_upload_moto(moto_s3): assert r.status_code == 204, r.text # Make sure the object is actually there - response = await moto_s3.list_objects(Bucket=BUCKET_NAME) - for obj in response["Contents"]: - assert obj["Key"] == key + get_info = await moto_s3.generate_presigned_url( + "get_object", + Params={"Bucket": BUCKET_NAME, "Key": key}, + ExpiresIn=3600, + ) + async with httpx.AsyncClient() as client: + r = await client.get( + get_info, + ) + + assert r.content == file_content @pytest.fixture(scope="function") From 5731226bcda2e5863806004ad5f023dd5b3ff7d5 Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Wed, 22 Apr 2026 14:52:20 +0200 Subject: [PATCH 4/5] test: fix signurlarity exception match --- diracx-logic/tests/jobs/test_sandboxes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/diracx-logic/tests/jobs/test_sandboxes.py b/diracx-logic/tests/jobs/test_sandboxes.py index 1558d6951..0dc3ddce9 100644 --- a/diracx-logic/tests/jobs/test_sandboxes.py +++ b/diracx-logic/tests/jobs/test_sandboxes.py @@ -142,7 +142,10 @@ async def test_upload_and_clean( await clean_sandboxes(sandbox_metadata_db, sandbox_settings) # Check that the sandbox was actually removed from the bucket - with pytest.raises(signurlarity.exceptions.NoSuchBucketError, match="Not Found"): + with pytest.raises( + signurlarity.exceptions.PresignError, + match="does not exist or is not accessible", + ): await sandbox_settings.s3_client.head_object( Bucket=sandbox_settings.bucket_name, Key=key ) From ca6f7345e1f461180e4a250786d5bc248e6e9130 Mon Sep 17 00:00:00 2001 From: natthan-pigoux Date: Fri, 24 Apr 2026 15:40:55 +0200 Subject: [PATCH 5/5] test: fix minio tests using signurlarity --- diracx-core/tests/test_s3.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/diracx-core/tests/test_s3.py b/diracx-core/tests/test_s3.py index e0ddd3531..c7b44e764 100644 --- a/diracx-core/tests/test_s3.py +++ b/diracx-core/tests/test_s3.py @@ -92,14 +92,14 @@ async def test_presigned_upload_moto(moto_s3): assert r.status_code == 204, r.text # Make sure the object is actually there - get_info = await moto_s3.generate_presigned_url( + url = await moto_s3.generate_presigned_url( "get_object", Params={"Bucket": BUCKET_NAME, "Key": key}, ExpiresIn=3600, ) async with httpx.AsyncClient() as client: r = await client.get( - get_info, + url, ) assert r.content == file_content @@ -123,8 +123,14 @@ async def test_bucket(minio_client): await minio_client.create_bucket(Bucket=bucket_name) yield bucket_name objects = await minio_client.list_objects(Bucket=bucket_name) - for obj in objects.get("Contents", []): - await minio_client.delete_object(Bucket=bucket_name, Key=obj["Key"]) + if objects.get("Contents", []): + await minio_client.delete_objects( + Bucket=bucket_name, + Delete={ + "Objects": [{"Key": obj["Key"]} for obj in objects.get("Contents", [])] + }, + ) + await minio_client.delete_bucket(Bucket=bucket_name)