diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1fd11f86d..06401f051 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,8 +49,6 @@ repos: - types-requests - types-python-dateutil - types-croniter - - types-aiobotocore[essential] - - boto3-stubs[essential] exclude: ^(diracx-client/src/diracx/client/_generated|diracx-[a-z]+/tests/|diracx-testing/|build|extensions/gubbins/gubbins-client/src/gubbins/client/_generated) - repo: https://github.com/hukkin/mdformat diff --git a/diracx-core/pyproject.toml b/diracx-core/pyproject.toml index 720968401..4e31ab77e 100644 --- a/diracx-core/pyproject.toml +++ b/diracx-core/pyproject.toml @@ -13,8 +13,6 @@ classifiers = [ "Topic :: System :: Distributed Computing", ] dependencies = [ - "aiobotocore>=2.15", - "botocore>=1.35", "cachetools", "email_validator", "gitpython", @@ -23,6 +21,7 @@ dependencies = [ "pydantic-settings", "pyyaml", "sh", + "signurlarity >=0.2.2", "diraccommon >=9.0.18", ] dynamic = ["version"] @@ -33,9 +32,6 @@ testing = [ "moto[server]", ] types = [ - "botocore-stubs", - "types-aiobotocore[essential]", - "types-aiobotocore-s3", "types-cachetools", "types-PyYAML", ] diff --git a/diracx-core/src/diracx/core/s3.py b/diracx-core/src/diracx/core/s3.py index afc0c9d20..bcebf5f9f 100644 --- a/diracx-core/src/diracx/core/s3.py +++ b/diracx-core/src/diracx/core/s3.py @@ -13,14 +13,14 @@ import base64 from typing import TYPE_CHECKING, TypedDict, cast -from botocore.errorfactory import ClientError +from signurlarity.exceptions import NoSuchBucketError, PresignError from .models.sandbox import ChecksumAlgorithm if TYPE_CHECKING: from typing import TypedDict - from types_aiobotocore_s3.client import S3Client + from signurlarity.aio.client import AsyncClient class S3Object(TypedDict): Key: str @@ -31,12 +31,12 @@ class S3PresignedPostInfo(TypedDict): fields: dict[str, str] -async def s3_bucket_exists(s3_client: S3Client, bucket_name: str) -> bool: +async def s3_bucket_exists(s3_client: AsyncClient, bucket_name: str) -> bool: """Check if a bucket exists in S3.""" return await _s3_exists(s3_client.head_bucket, Bucket=bucket_name) -async def s3_object_exists(s3_client: S3Client, bucket_name: str, key: str) -> bool: +async def s3_object_exists(s3_client: AsyncClient, bucket_name: str, key: str) -> bool: """Check if an object exists in an S3 bucket.""" return await _s3_exists(s3_client.head_object, Bucket=bucket_name, Key=key) @@ -44,16 +44,46 @@ async def s3_object_exists(s3_client: S3Client, bucket_name: str, key: str) -> b async def _s3_exists(method, **kwargs: str) -> bool: try: await method(**kwargs) - except ClientError as e: - if e.response["Error"]["Code"] != "404": - raise + except (NoSuchBucketError, PresignError): + # if e.response["Error"]["Code"] != "404": + # raise return False else: return True async def generate_presigned_upload( - s3_client: S3Client, + s3_client: AsyncClient, + bucket_name: str, + key: str, + checksum_algorithm: ChecksumAlgorithm, + checksum: str, + size: int, + validity_seconds: int, +) -> S3PresignedPostInfo: + """Generate a presigned URL and fields for uploading a file to S3. + + The signature is restricted to only accept data with the given checksum and size. + """ + fields = { + "x-amz-checksum-algorithm": checksum_algorithm, + f"x-amz-checksum-{checksum_algorithm}": b16_to_b64(checksum), + } + conditions = [["content-length-range", size, size]] + [ + {k: v} for k, v in fields.items() + ] + result = await s3_client.generate_presigned_post( + Bucket=bucket_name, + Key=key, + Fields=fields, + Conditions=conditions, + ExpiresIn=validity_seconds, + ) + return cast(S3PresignedPostInfo, result) + + +async def generate_presigned_download( + s3_client: AsyncClient, bucket_name: str, key: str, checksum_algorithm: ChecksumAlgorithm, @@ -126,7 +156,7 @@ async def _s3_delete_chunk_with_retry( Bucket=bucket, Delete={"Objects": remaining, "Quiet": True}, ) - except ClientError: + except NoSuchBucketError: if attempt == max_attempts: return {obj["Key"] for obj in remaining} await asyncio.sleep(delay) diff --git a/diracx-core/src/diracx/core/settings.py b/diracx-core/src/diracx/core/settings.py index 40c75afd2..93a292975 100644 --- a/diracx-core/src/diracx/core/settings.py +++ b/diracx-core/src/diracx/core/settings.py @@ -18,9 +18,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Annotated, Any, Self, TypeVar, cast -from aiobotocore.session import get_session -from botocore.config import Config -from botocore.errorfactory import ClientError from cryptography.fernet import Fernet from joserfc.jwk import KeySet, KeySetSerialization from pydantic import ( @@ -35,9 +32,11 @@ model_validator, ) from pydantic_settings import BaseSettings, SettingsConfigDict +from signurlarity.aio.client import AsyncClient +from signurlarity.exceptions import NoSuchBucketError if TYPE_CHECKING: - from types_aiobotocore_s3.client import S3Client + from signurlarity.aio.client import AsyncClient T = TypeVar("T") @@ -324,17 +323,12 @@ class SandboxStoreSettings(ServiceSettingsBase): Controls parallelism of database DELETE operations. """ - _client: S3Client = PrivateAttr() + _client: AsyncClient = PrivateAttr() @contextlib.asynccontextmanager async def lifetime_function(self) -> AsyncIterator[None]: - async with get_session().create_client( - "s3", + async with AsyncClient( **self.s3_client_kwargs, - config=Config( - signature_version="v4", - max_pool_connections=self.s3_max_pool_connections, - ), ) as self._client: # type: ignore if not await s3_bucket_exists(self._client, self.bucket_name): if not self.auto_create_bucket: @@ -343,7 +337,7 @@ async def lifetime_function(self) -> AsyncIterator[None]: ) try: await self._client.create_bucket(Bucket=self.bucket_name) - except ClientError as e: + except NoSuchBucketError as e: raise ValueError( f"Failed to create bucket {self.bucket_name}" ) from e @@ -351,7 +345,7 @@ async def lifetime_function(self) -> AsyncIterator[None]: yield @property - def s3_client(self) -> S3Client: + def s3_client(self) -> AsyncClient: if self._client is None: raise RuntimeError("S3 client accessed before lifetime function") return self._client diff --git a/diracx-core/tests/test_s3.py b/diracx-core/tests/test_s3.py index 42f6ea022..c7b44e764 100644 --- a/diracx-core/tests/test_s3.py +++ b/diracx-core/tests/test_s3.py @@ -7,7 +7,7 @@ import httpx import pytest -from aiobotocore.session import get_session +from signurlarity.aio.client import AsyncClient from diracx.core.s3 import ( b16_to_b64, @@ -52,7 +52,7 @@ async def moto_s3(aio_moto): Note that this is not a complete S3 backend, in particular authentication and validation of requests is not implemented. """ - async with get_session().create_client("s3", **aio_moto) as client: + async with AsyncClient(**aio_moto) as client: await client.create_bucket(Bucket=BUCKET_NAME) await client.create_bucket(Bucket=OTHER_BUCKET_NAME) yield client @@ -92,15 +92,23 @@ async def test_presigned_upload_moto(moto_s3): assert r.status_code == 204, r.text # Make sure the object is actually there - obj = await moto_s3.get_object(Bucket=BUCKET_NAME, Key=key) - assert (await obj["Body"].read()) == file_content + url = await moto_s3.generate_presigned_url( + "get_object", + Params={"Bucket": BUCKET_NAME, "Key": key}, + ExpiresIn=3600, + ) + async with httpx.AsyncClient() as client: + r = await client.get( + url, + ) + + assert r.content == file_content @pytest.fixture(scope="function") async def minio_client(demo_urls): """Create a S3 client that uses minio from the demo as backend.""" - async with get_session().create_client( - "s3", + async with AsyncClient( endpoint_url=demo_urls["minio"], aws_access_key_id="console", aws_secret_access_key="console123", @@ -115,8 +123,14 @@ async def test_bucket(minio_client): await minio_client.create_bucket(Bucket=bucket_name) yield bucket_name objects = await minio_client.list_objects(Bucket=bucket_name) - for obj in objects.get("Contents", []): - await minio_client.delete_object(Bucket=bucket_name, Key=obj["Key"]) + if objects.get("Contents", []): + await minio_client.delete_objects( + Bucket=bucket_name, + Delete={ + "Objects": [{"Key": obj["Key"]} for obj in objects.get("Contents", [])] + }, + ) + await minio_client.delete_bucket(Bucket=bucket_name) diff --git a/diracx-logic/tests/jobs/test_sandboxes.py b/diracx-logic/tests/jobs/test_sandboxes.py index 6dc59197d..0dc3ddce9 100644 --- a/diracx-logic/tests/jobs/test_sandboxes.py +++ b/diracx-logic/tests/jobs/test_sandboxes.py @@ -6,10 +6,10 @@ from io import BytesIO from typing import AsyncGenerator, Generator -import botocore.exceptions import freezegun import httpx import pytest +import signurlarity.exceptions import sqlalchemy from diracx.core.exceptions import SandboxNotFoundError @@ -142,7 +142,10 @@ async def test_upload_and_clean( await clean_sandboxes(sandbox_metadata_db, sandbox_settings) # Check that the sandbox was actually removed from the bucket - with pytest.raises(botocore.exceptions.ClientError, match="Not Found"): + with pytest.raises( + signurlarity.exceptions.PresignError, + match="does not exist or is not accessible", + ): await sandbox_settings.s3_client.head_object( Bucket=sandbox_settings.bucket_name, Key=key ) diff --git a/extensions/gubbins/gubbins-logic/pyproject.toml b/extensions/gubbins/gubbins-logic/pyproject.toml index af465c4b2..4f7c9a3de 100644 --- a/extensions/gubbins/gubbins-logic/pyproject.toml +++ b/extensions/gubbins/gubbins-logic/pyproject.toml @@ -23,9 +23,6 @@ dynamic = ["version"] [project.optional-dependencies] testing = [] types = [ - "boto3-stubs", - "types-aiobotocore[essential]", - "types-aiobotocore-s3", "types-cachetools", "types-python-dateutil", "types-PyYAML", diff --git a/extensions/gubbins/gubbins-routers/pyproject.toml b/extensions/gubbins/gubbins-routers/pyproject.toml index ffc5313e4..e339729e1 100644 --- a/extensions/gubbins/gubbins-routers/pyproject.toml +++ b/extensions/gubbins/gubbins-routers/pyproject.toml @@ -23,9 +23,6 @@ dynamic = ["version"] [project.optional-dependencies] testing = ["diracx-testing", "moto[server]", "pytest-httpx"] types = [ - "boto3-stubs", - "types-aiobotocore[essential]", - "types-aiobotocore-s3", "types-cachetools", "types-python-dateutil", "types-PyYAML",