From d6a9b4f2461141da512716c7e922a9f647088804 Mon Sep 17 00:00:00 2001 From: raytesnel Date: Tue, 10 Mar 2026 10:57:45 +0100 Subject: [PATCH 1/9] remove overrides for swagger docs --- src/preprocessors/router.py | 36 ++--------------------- src/preprocessors/schemas.py | 55 ------------------------------------ 2 files changed, 3 insertions(+), 88 deletions(-) diff --git a/src/preprocessors/router.py b/src/preprocessors/router.py index 426180f6..c039294c 100644 --- a/src/preprocessors/router.py +++ b/src/preprocessors/router.py @@ -37,33 +37,6 @@ preprocessor_route = APIRouter(prefix=f"/{RoutePrefix.PREPROCESSOR}", tags=[RoutePrefix.PREPROCESSOR]) -def _generate_openapi_schema(model: type[BaseModel]) -> dict[str, Any]: - """Generate example fields in the Swagger docs for endpoints receiving multipart/form-data with a binary mask.""" - return { - "requestBody": { - "content": { - "multipart/form-data": { - "schema": { - "properties": { - "params": model.model_json_schema(), - "mask_data": {"type": "string", "format": "binary", "example": b"\x01\x00\x00\x01"}, - }, - "required": ["params", "mask_data"], - } - }, - "application/json": { - "schema": { - "properties": { - "params": model.model_json_schema(), - }, - "required": ["params"], - } - }, - } - } - } - - @preprocessor_route.get( path=PreprocessorEndpoint.ROOT, summary="Redirect to preprocessor documentation", @@ -135,10 +108,9 @@ async def process_scan(upload_scan: UploadScan) -> ProcessedDataAccess: HTTPStatus.UNPROCESSABLE_ENTITY: {"description": "mask shape does not match image shape"}, HTTPStatus.INTERNAL_SERVER_ERROR: {"description": "image generation error"}, }, - openapi_extra=_generate_openapi_schema(model=PrepareMarkImpression), ) async def prepare_mark_impression( - params: Annotated[Json[PrepareMarkImpression], Form(...)], mask_data: bytes = File(...) + params: Annotated[PrepareMarkImpression, Form()], mask_data: Annotated[bytes, File()] ) -> PrepareMarkResponseImpression: """Prepare the ScanFile, save it to the vault and return the URLs to access the files.""" vault = create_vault(params.tag) @@ -181,10 +153,9 @@ async def prepare_mark_impression( HTTPStatus.UNPROCESSABLE_ENTITY: {"description": "mask shape does not match image shape"}, HTTPStatus.INTERNAL_SERVER_ERROR: {"description": "image generation error"}, }, - openapi_extra=_generate_openapi_schema(model=PrepareMarkStriation), ) async def prepare_mark_striation( - params: Annotated[Json[PrepareMarkStriation], Form(...)], mask_data: bytes = File(...) + params: Annotated[PrepareMarkStriation, Form()], mask_data: Annotated[bytes, File()] ) -> PrepareMarkResponseStriation: """Prepare the ScanFile, save it to the vault and return the URLs to access the files.""" vault = create_vault(params.tag) @@ -230,9 +201,8 @@ async def prepare_mark_striation( "description": "processing error", }, }, - openapi_extra=_generate_openapi_schema(model=EditImage), ) -async def edit_scan(params: Annotated[Json[EditImage], Form(...)], mask_data: bytes = File(...)) -> GeneratedImages: +async def edit_scan(params: Annotated[EditImage, Form()], mask_data: Annotated[bytes, File()]) -> GeneratedImages: """ Validate and parse a scan file with edit parameters and mask. diff --git a/src/preprocessors/schemas.py b/src/preprocessors/schemas.py index 2dfd3c63..c15f7612 100644 --- a/src/preprocessors/schemas.py +++ b/src/preprocessors/schemas.py @@ -27,17 +27,6 @@ from schemas import URLContainer -def _update_schema(schema: dict[str, Any], attr_to_class: tuple[tuple[str, str], ...]) -> dict[str, Any]: - """Update the model JSON schema for correctly rendering the `openapi_extra` fields.""" - for attribute, class_name in attr_to_class: - updated = schema["$defs"][class_name] - for key in ("examples", "description"): - if value := schema["properties"][attribute].get(key): - updated[key] = value - schema["properties"][attribute] = updated - return schema - - class BaseParameters(BaseModelConfig): """Base parameters for preprocessor operations including scan file.""" @@ -59,16 +48,6 @@ def tag(self) -> str: """Get the tag to use for directory naming.""" return self.project_name or self.scan_file.stem - @classmethod - def model_json_schema(cls, *args, **kwargs) -> dict[str, Any]: - """Override the base method.""" - schema = super().model_json_schema(*args, **kwargs) - attr_to_class = ( - ("scan_file", "ScanFile"), - ("project_name", "ProjectTag"), - ) - return _update_schema(schema, attr_to_class) - class UploadScan(BaseParameters): scale_x: PositiveFloat = Field( @@ -112,13 +91,6 @@ def bounding_box(self) -> BoundingBox | None: """ return np.array(self.bounding_box_list) if self.bounding_box_list is not None else None - @classmethod - def model_json_schema(cls, *args, **kwargs) -> dict[str, Any]: - """Override the base method.""" - schema = super().model_json_schema(*args, **kwargs) - attr_to_class = (("mark_type", "MarkType"),) - return _update_schema(schema, attr_to_class) - class PrepareMarkStriation(PrepareMarkBase): mark_parameters: PreprocessingStriationParams = Field(..., description="Preprocessor parameters.") @@ -131,13 +103,6 @@ def must_be_striation(cls, v: MarkType) -> MarkType: raise ValueError(f"{v} is not a striation mark") return v - @classmethod - def model_json_schema(cls, *args, **kwargs) -> dict[str, Any]: - """Override the base method.""" - schema = super().model_json_schema(*args, **kwargs) - attr_to_class = (("mark_parameters", "PreprocessingStriationParams"),) - return _update_schema(schema, attr_to_class) - class PrepareMarkImpression(PrepareMarkBase): mark_parameters: PreprocessingImpressionParams = Field(..., description="Preprocessor parameters.") @@ -150,13 +115,6 @@ def must_be_impression(cls, v: MarkType) -> MarkType: raise ValueError(f"{v} is not an impression mark") return v - @classmethod - def model_json_schema(cls, *args, **kwargs) -> dict[str, Any]: - """Override the base method.""" - schema = super().model_json_schema(*args, **kwargs) - attr_to_class = (("mark_parameters", "PreprocessingImpressionParams"),) - return _update_schema(schema, attr_to_class) - class EditImage(BaseParameters): """Request model for editing and transforming processed scan images.""" @@ -198,19 +156,6 @@ def check_file_is_x3p(self): if self.scan_file.suffix.lower() != ".x3p": raise ValueError(f"Unsupported extension: {self.scan_file.suffix}") return self - - @classmethod - def model_json_schema(cls, *args, **kwargs) -> dict[str, Any]: - """Override the base method.""" - schema = super().model_json_schema(*args, **kwargs) - # Add schema for BaseParameters and EditImage to JSON model - attr_to_class = ( - ("regression_order", "RegressionOrder"), - ("terms", "SurfaceOptions"), - ) - return _update_schema(schema, attr_to_class) - - class GeneratedImages(URLContainer): preview_image: HttpUrl = Field( ..., From d43ce5244e2eff91a0fa363f29e3eaf410f449a4 Mon Sep 17 00:00:00 2001 From: raytesnel Date: Tue, 10 Mar 2026 16:00:36 +0100 Subject: [PATCH 2/9] remove overrides for swagger docs --- src/preprocessors/router.py | 30 +++++++++-------------- src/preprocessors/schemas.py | 29 ++++++++++++++++++---- tests/preprocessors/router/test_router.py | 20 ++++++++------- 3 files changed, 47 insertions(+), 32 deletions(-) diff --git a/src/preprocessors/router.py b/src/preprocessors/router.py index c039294c..65d88db1 100644 --- a/src/preprocessors/router.py +++ b/src/preprocessors/router.py @@ -1,10 +1,8 @@ from http import HTTPStatus -from typing import Annotated, Any -from fastapi import APIRouter, File, Form, HTTPException +from fastapi import APIRouter, Body, Depends, File, Form, HTTPException, UploadFile from fastapi.responses import RedirectResponse from loguru import logger -from pydantic import BaseModel, Json from constants import ( LIGHT_SOURCES, @@ -109,16 +107,14 @@ async def process_scan(upload_scan: UploadScan) -> ProcessedDataAccess: HTTPStatus.INTERNAL_SERVER_ERROR: {"description": "image generation error"}, }, ) -async def prepare_mark_impression( - params: Annotated[PrepareMarkImpression, Form()], mask_data: Annotated[bytes, File()] -) -> PrepareMarkResponseImpression: - """Prepare the ScanFile, save it to the vault and return the URLs to access the files.""" +async def prepare_mark_impression(params: PrepareMarkImpression = Depends()) -> PrepareMarkResponseImpression: + """Prepare the ScanFile, save it to the vault and return the urls to acces the files.""" vault = create_vault(params.tag) parsed_image = parse_scan_pipeline(params.scan_file, 1, 1) - + byte_contents = await params.mask_data.read() try: parsed_mask = parse_mask_pipeline( - raw_data=mask_data, + raw_data=byte_contents, shape=parsed_image.data.shape, is_bitpacked=params.mask_is_bitpacked, ) @@ -154,16 +150,14 @@ async def prepare_mark_impression( HTTPStatus.INTERNAL_SERVER_ERROR: {"description": "image generation error"}, }, ) -async def prepare_mark_striation( - params: Annotated[PrepareMarkStriation, Form()], mask_data: Annotated[bytes, File()] -) -> PrepareMarkResponseStriation: - """Prepare the ScanFile, save it to the vault and return the URLs to access the files.""" +async def prepare_mark_striation(params: PrepareMarkStriation = Depends()) -> PrepareMarkResponseStriation: + """Prepare the ScanFile, save it to the vault and return the urls to acces the files.""" vault = create_vault(params.tag) parsed_image = parse_scan_pipeline(params.scan_file, 1, 1) - + byte_contents = await params.mask_data.read() try: parsed_mask = parse_mask_pipeline( - raw_data=mask_data, + raw_data=byte_contents, shape=parsed_image.data.shape, is_bitpacked=params.mask_is_bitpacked, ) @@ -202,7 +196,7 @@ async def prepare_mark_striation( }, }, ) -async def edit_scan(params: Annotated[EditImage, Form()], mask_data: Annotated[bytes, File()]) -> GeneratedImages: +async def edit_scan(params: EditImage = Depends()) -> GeneratedImages: """ Validate and parse a scan file with edit parameters and mask. @@ -213,10 +207,10 @@ async def edit_scan(params: Annotated[EditImage, Form()], mask_data: Annotated[b vault = create_vault(params.tag) logger.debug(f"Working directory created on: {vault.resource_path}") parsed_image = parse_scan_pipeline(params.scan_file, 1, 1) - + byte_contents = await params.mask_data.read() try: parsed_mask = parse_mask_pipeline( - raw_data=mask_data, + raw_data=byte_contents, shape=parsed_image.data.shape, is_bitpacked=params.mask_is_bitpacked, ) diff --git a/src/preprocessors/schemas.py b/src/preprocessors/schemas.py index c15f7612..00a72123 100644 --- a/src/preprocessors/schemas.py +++ b/src/preprocessors/schemas.py @@ -1,12 +1,13 @@ from __future__ import annotations from functools import cached_property -from typing import Any +from typing import Annotated,Any import numpy as np from conversion.data_formats import BoundingBox, MarkType from conversion.preprocess_impression.parameters import PreprocessingImpressionParams from conversion.preprocess_striation import PreprocessingStriationParams +from fastapi import File, Form, UploadFile from pydantic import ( Field, HttpUrl, @@ -70,8 +71,8 @@ class UploadScan(BaseParameters): class PrepareMarkBase(BaseParameters): mark_type: MarkType = Field(..., description="Type of mark to prepare.") - bounding_box_list: list[list[float]] | None = Field( - None, + bounding_box_list: list[float] | None = Field( + default_factory=list, description="Bounding box corners (4 × 2 array of [x, y] coordinates) " "defining a rectangular crop region used to determine the rotation of the image.", ) @@ -93,7 +94,14 @@ def bounding_box(self) -> BoundingBox | None: class PrepareMarkStriation(PrepareMarkBase): - mark_parameters: PreprocessingStriationParams = Field(..., description="Preprocessor parameters.") + highpass_cutoff: float = 2e-3 + lowpass_cutoff: float = 2.5e-4 + cut_borders_after_smoothing: bool = True + use_mean: bool = True + angle_accuracy: float = 0.1 + max_iter: int = 25 + subsampling_factor: int = 1 + mask_data: UploadFile = File() @field_validator("mark_type") @classmethod @@ -105,7 +113,17 @@ def must_be_striation(cls, v: MarkType) -> MarkType: class PrepareMarkImpression(PrepareMarkBase): - mark_parameters: PreprocessingImpressionParams = Field(..., description="Preprocessor parameters.") + pixel_size: float | None = None + adjust_pixel_spacing: bool = True + level_offset: bool = True + level_tilt: bool = True + level_2nd: bool = True + interp_method: str = "cubic" + highpass_cutoff: float | None = 250.0e-6 + lowpass_cutoff: float | None = 5.0e-6 + highpass_regression_order: int = 2 + lowpass_regression_order: int = 0 + mask_data: UploadFile = File() @field_validator("mark_type") @classmethod @@ -149,6 +167,7 @@ class EditImage(BaseParameters): 'The expected bit-order for bit-packed arrays is "little".', examples=[True, False], ) + mask_data: UploadFile = File() @model_validator(mode="after") def check_file_is_x3p(self): diff --git a/tests/preprocessors/router/test_router.py b/tests/preprocessors/router/test_router.py index 3e2cb12f..8c9f10fa 100644 --- a/tests/preprocessors/router/test_router.py +++ b/tests/preprocessors/router/test_router.py @@ -31,8 +31,10 @@ def send_post_request_with_mask(client: TestClient, endpoint: str, params: dict, mask: BinaryMask) -> Response: return client.post( f"{get_settings().base_url}/{RoutePrefix.PREPROCESSOR}/{endpoint}", - data={"params": json.dumps(params, default=str)}, - files={"mask_data": ("mask.bin", mask.tobytes(order="C"), "application/octet-stream")}, + params=params, + files={ + "mask_data": ("mask.bin", mask.tobytes(order="C"), "application/octet-stream"), + }, timeout=5, ) @@ -111,12 +113,11 @@ def get_schema_for_endpoint( mark_parameters: type[PreprocessingStriationParams | PreprocessingImpressionParams], ): """Generate the schema payload for the prepare-mark endpoint.""" - return schema( + return schema.model_construct( project_name="test_project", mark_type=mark_type, # type: ignore scan_file=self.scan_file_path, bounding_box_list=[[1.0, 1.0], [10.0, 1.0], [10.0, 10.0], [1.0, 10.0]], - mark_parameters=mark_parameters(), # type: ignore ).model_dump(mode="json") def test_prepare_mark_endpoint_returns_urls( # noqa: PLR0913 @@ -133,12 +134,12 @@ def test_prepare_mark_endpoint_returns_urls( # noqa: PLR0913 ) -> None: """Test that the prepare-mark endpoint processes the request and returns file URLs.""" # Arrange - payload = self.get_schema_for_endpoint( + payload: dict = self.get_schema_for_endpoint( schema=schema, mark_type=mark_type, mark_parameters=mark_parameters, ) - + payload.pop("bounding_box_list") # Act response = send_post_request_with_mask(client=client, endpoint=endpoint, params=payload, mask=mask) @@ -274,7 +275,7 @@ def test_edit_image_returns_valid_images( mask = np.zeros(shape=(259, 259), dtype=np.bool_) mask[1:259, 1:259] = True - params = EditImage( + params = EditImage.model_construct( project_name="test", scan_file=scan_directory / "circle.x3p", cutoff_length=2 * micro, @@ -288,12 +289,13 @@ def test_edit_image_returns_valid_images( with monkeypatch.context() as mp: mp.setattr("preprocessors.router.create_vault", lambda _: directory_access) response = send_post_request_with_mask(client=client, endpoint="edit-scan", params=params, mask=mask) - # Assert + expected_response = GeneratedImages( preview_image=HttpUrl(f"{base_url}/preview.png"), surface_map_image=HttpUrl(f"{base_url}/surface_map.png"), ) - assert response.status_code == HTTPStatus.OK, "endpoint is alive" + # Assert + assert response.status_code == HTTPStatus.OK, response.text response_model = GeneratedImages.model_validate(response.json()) assert response_model == expected_response assert (directory / "preview.png").exists() From b1f55957ff7e5b4a43cf4eb7efe771b16c6b8196 Mon Sep 17 00:00:00 2001 From: raytesnel Date: Thu, 26 Mar 2026 17:15:06 +0100 Subject: [PATCH 3/9] fix boundingbox typing --- src/preprocessors/router.py | 2 +- src/preprocessors/schemas.py | 13 ++++++------- tests/preprocessors/router/test_router.py | 1 - 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/preprocessors/router.py b/src/preprocessors/router.py index 65d88db1..0aa48d22 100644 --- a/src/preprocessors/router.py +++ b/src/preprocessors/router.py @@ -1,6 +1,6 @@ from http import HTTPStatus -from fastapi import APIRouter, Body, Depends, File, Form, HTTPException, UploadFile +from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import RedirectResponse from loguru import logger diff --git a/src/preprocessors/schemas.py b/src/preprocessors/schemas.py index 00a72123..677c857e 100644 --- a/src/preprocessors/schemas.py +++ b/src/preprocessors/schemas.py @@ -1,13 +1,10 @@ from __future__ import annotations from functools import cached_property -from typing import Annotated,Any import numpy as np from conversion.data_formats import BoundingBox, MarkType -from conversion.preprocess_impression.parameters import PreprocessingImpressionParams -from conversion.preprocess_striation import PreprocessingStriationParams -from fastapi import File, Form, UploadFile +from fastapi import File, UploadFile from pydantic import ( Field, HttpUrl, @@ -70,9 +67,9 @@ class UploadScan(BaseParameters): class PrepareMarkBase(BaseParameters): - mark_type: MarkType = Field(..., description="Type of mark to prepare.") - bounding_box_list: list[float] | None = Field( - default_factory=list, + mark_type: MarkType + bounding_box_list: list[list[float]] | None = Field( + None, description="Bounding box corners (4 × 2 array of [x, y] coordinates) " "defining a rectangular crop region used to determine the rotation of the image.", ) @@ -175,6 +172,8 @@ def check_file_is_x3p(self): if self.scan_file.suffix.lower() != ".x3p": raise ValueError(f"Unsupported extension: {self.scan_file.suffix}") return self + + class GeneratedImages(URLContainer): preview_image: HttpUrl = Field( ..., diff --git a/tests/preprocessors/router/test_router.py b/tests/preprocessors/router/test_router.py index 8c9f10fa..927762c7 100644 --- a/tests/preprocessors/router/test_router.py +++ b/tests/preprocessors/router/test_router.py @@ -1,4 +1,3 @@ -import json from http import HTTPStatus from pathlib import Path From b06d89ebc6b9a6166976788a97e3d569c282089b Mon Sep 17 00:00:00 2001 From: raytesnel Date: Mon, 30 Mar 2026 07:05:19 +0200 Subject: [PATCH 4/9] fix flattened params useages --- .../preprocess_impression.py | 23 ++++++++++++--- .../preprocess_striation/pipeline.py | 14 +++++++-- .../test_striation_matlab.py | 9 +++--- .../tests/conversion/test_impression.py | 29 ++++++++++--------- src/preprocessors/controller.py | 10 +++---- src/preprocessors/router.py | 10 ++++--- 6 files changed, 62 insertions(+), 33 deletions(-) diff --git a/packages/scratch-core/src/conversion/preprocess_impression/preprocess_impression.py b/packages/scratch-core/src/conversion/preprocess_impression/preprocess_impression.py index be2fa57a..f9098a4b 100644 --- a/packages/scratch-core/src/conversion/preprocess_impression/preprocess_impression.py +++ b/packages/scratch-core/src/conversion/preprocess_impression/preprocess_impression.py @@ -6,28 +6,43 @@ from dataclasses import asdict +from pydantic import BaseModel + from container_models.base import DepthData from conversion.data_formats import Mark from conversion.filter import ( - apply_gaussian_filter_mark, apply_filter_pipeline, + apply_gaussian_filter_mark, ) from conversion.leveling import SurfaceTerms, level_map from conversion.mask import crop_to_mask from conversion.preprocess_impression.center import compute_center_local from conversion.preprocess_impression.parameters import PreprocessingImpressionParams from conversion.preprocess_impression.resample import ( - resample, needs_resampling, + resample, ) from conversion.preprocess_impression.tilt import apply_tilt_correction -from conversion.preprocess_impression.utils import update_mark_data, Point2D +from conversion.preprocess_impression.utils import Point2D, update_mark_data from conversion.resample import get_scaling_factors, resample_array_2d +class ImpressionParams(BaseModel): + pixel_size: float | None = None + adjust_pixel_spacing: bool = True + level_offset: bool = True + level_tilt: bool = True + level_2nd: bool = True + interp_method: str = "cubic" + highpass_cutoff: float | None = 250.0e-6 + lowpass_cutoff: float | None = 5.0e-6 + highpass_regression_order: int = 2 + lowpass_regression_order: int = 0 + + def preprocess_impression_mark( mark: Mark, - params: PreprocessingImpressionParams, + params: ImpressionParams, ) -> tuple[Mark, Mark]: """ Preprocess trimmed impression image data. diff --git a/packages/scratch-core/src/conversion/preprocess_striation/pipeline.py b/packages/scratch-core/src/conversion/preprocess_striation/pipeline.py index 45c56a39..8627f006 100644 --- a/packages/scratch-core/src/conversion/preprocess_striation/pipeline.py +++ b/packages/scratch-core/src/conversion/preprocess_striation/pipeline.py @@ -9,13 +9,14 @@ from dataclasses import asdict import numpy as np +from pydantic import BaseModel from container_models.base import FloatArray2D from container_models.scan_image import ScanImage from conversion.data_formats import Mark from conversion.filter import ( - cutoff_to_gaussian_sigma, apply_striation_preserving_filter_1d, + cutoff_to_gaussian_sigma, ) from conversion.preprocess_striation import PreprocessingStriationParams from conversion.preprocess_striation.alignment import fine_align_bullet_marks @@ -23,9 +24,18 @@ from conversion.profile_correlator import Profile +class StriationParams(BaseModel): + highpass_cutoff: float = 2e-3 + lowpass_cutoff: float = 2.5e-4 + cut_borders_after_smoothing: bool = True + use_mean: bool = True + angle_accuracy: float = 0.1 + max_iter: int = 25 + subsampling_factor: int = 1 + def preprocess_striation_mark( mark: Mark, - params: PreprocessingStriationParams, + params: StriationParams, ) -> tuple[Mark, Profile]: """ Complete the preprocessing pipeline for striated marks. diff --git a/packages/scratch-core/tests/conversion/preprocess_striation/test_striation_matlab.py b/packages/scratch-core/tests/conversion/preprocess_striation/test_striation_matlab.py index 5a6bc54a..91944548 100755 --- a/packages/scratch-core/tests/conversion/preprocess_striation/test_striation_matlab.py +++ b/packages/scratch-core/tests/conversion/preprocess_striation/test_striation_matlab.py @@ -12,18 +12,19 @@ from container_models.base import DepthData from conversion.data_formats import MarkType -from ..helper_functions import make_mark from conversion.preprocess_striation import ( PreprocessingStriationParams, preprocess_striation_mark, ) +from conversion.preprocess_striation.pipeline import StriationParams + from ..helper_functions import ( _compute_correlation, - _crop_to_common_shape, _compute_difference_stats, + _crop_to_common_shape, + make_mark, ) - MARK_TYPE_MAPPING = { "bullet lea striation": MarkType.BULLET_LEA_STRIATION, "bullet gea striation": MarkType.BULLET_GEA_STRIATION, @@ -166,7 +167,7 @@ def run_python_preprocessing( test_case: MatlabTestCase, ) -> tuple[np.ndarray, np.ndarray | None, float | None]: """Run Python preprocess_striation_mark and return the results.""" - params = PreprocessingStriationParams( + params = StriationParams( highpass_cutoff=test_case.cutoff_hi * micro, lowpass_cutoff=test_case.cutoff_lo * micro, use_mean=test_case.use_mean, diff --git a/packages/scratch-core/tests/conversion/test_impression.py b/packages/scratch-core/tests/conversion/test_impression.py index ac8b4087..2067e22b 100644 --- a/packages/scratch-core/tests/conversion/test_impression.py +++ b/packages/scratch-core/tests/conversion/test_impression.py @@ -17,6 +17,7 @@ ) from conversion.preprocess_impression.parameters import PreprocessingImpressionParams from conversion.preprocess_impression.preprocess_impression import ( + ImpressionParams, preprocess_impression_mark, ) from conversion.preprocess_impression.resample import needs_resampling @@ -422,7 +423,7 @@ def test_basic_pipeline_runs(self): scale_y=micro, mark_type=MarkType.FIRING_PIN_IMPRESSION, ) - params = PreprocessingImpressionParams( + params = ImpressionParams( pixel_size=2 * micro, adjust_pixel_spacing=False, level_offset=True, @@ -446,7 +447,7 @@ def test_output_has_correct_scale(self): mark_type=MarkType.FIRING_PIN_IMPRESSION, ) target_size = 2 * micro - params = PreprocessingImpressionParams( + params = ImpressionParams( pixel_size=target_size, adjust_pixel_spacing=False, ) @@ -468,7 +469,7 @@ def test_output_is_smaller_after_downsampling(self): scale_y=micro, mark_type=MarkType.FIRING_PIN_IMPRESSION, ) - params = PreprocessingImpressionParams( + params = ImpressionParams( pixel_size=2 * micro, # 2x downsampling adjust_pixel_spacing=False, ) @@ -491,7 +492,7 @@ def test_filtered_and_leveled_differ(self): scale_y=micro, mark_type=MarkType.FIRING_PIN_IMPRESSION, ) - params = PreprocessingImpressionParams( + params = ImpressionParams( adjust_pixel_spacing=False, highpass_cutoff=50 * micro, # Apply high-pass to create difference ) @@ -514,7 +515,7 @@ def test_breech_face_uses_circle_center(self): scale_y=micro, mark_type=MarkType.BREECH_FACE_IMPRESSION, ) - params = PreprocessingImpressionParams( + params = ImpressionParams( pixel_size=micro, # No resampling adjust_pixel_spacing=False, ) @@ -535,7 +536,7 @@ def test_no_resampling_when_pixel_size_matches(self): scale_y=micro, mark_type=MarkType.FIRING_PIN_IMPRESSION, ) - params = PreprocessingImpressionParams( + params = ImpressionParams( pixel_size=micro, # Same as input adjust_pixel_spacing=False, ) @@ -555,7 +556,7 @@ def test_is_resampled_flag_set_on_resampling(self): scale_y=micro, mark_type=MarkType.FIRING_PIN_IMPRESSION, ) - params = PreprocessingImpressionParams( + params = ImpressionParams( pixel_size=2 * micro, # Different from input adjust_pixel_spacing=False, ) @@ -575,7 +576,7 @@ def test_without_lowpass_filter(self): scale_y=micro, mark_type=MarkType.FIRING_PIN_IMPRESSION, ) - params = PreprocessingImpressionParams( + params = ImpressionParams( pixel_size=2 * micro, adjust_pixel_spacing=False, lowpass_cutoff=None, @@ -596,7 +597,7 @@ def test_without_highpass_filter(self): scale_y=micro, mark_type=MarkType.FIRING_PIN_IMPRESSION, ) - params = PreprocessingImpressionParams( + params = ImpressionParams( pixel_size=2 * micro, adjust_pixel_spacing=False, highpass_cutoff=None, @@ -617,7 +618,7 @@ def test_without_any_filters(self): scale_y=micro, mark_type=MarkType.FIRING_PIN_IMPRESSION, ) - params = PreprocessingImpressionParams( + params = ImpressionParams( pixel_size=2 * micro, adjust_pixel_spacing=False, lowpass_cutoff=None, @@ -639,7 +640,7 @@ def test_with_tilt_adjustment(self): scale_y=micro, mark_type=MarkType.FIRING_PIN_IMPRESSION, ) - params = PreprocessingImpressionParams( + params = ImpressionParams( pixel_size=2 * micro, adjust_pixel_spacing=True, ) @@ -659,7 +660,7 @@ def test_with_second_order_leveling(self): scale_y=micro, mark_type=MarkType.FIRING_PIN_IMPRESSION, ) - params = PreprocessingImpressionParams( + params = ImpressionParams( pixel_size=2 * micro, adjust_pixel_spacing=False, level_offset=True, @@ -682,7 +683,7 @@ def test_output_data_is_finite_where_valid(self): scale_y=micro, mark_type=MarkType.FIRING_PIN_IMPRESSION, ) - params = PreprocessingImpressionParams( + params = ImpressionParams( pixel_size=2 * micro, adjust_pixel_spacing=False, ) @@ -715,7 +716,7 @@ def test_leveled_preserves_form(self): scale_y=micro, mark_type=MarkType.FIRING_PIN_IMPRESSION, ) - params = PreprocessingImpressionParams( + params = ImpressionParams( pixel_size=micro, adjust_pixel_spacing=False, level_offset=True, diff --git a/src/preprocessors/controller.py b/src/preprocessors/controller.py index 58917721..0934fd52 100644 --- a/src/preprocessors/controller.py +++ b/src/preprocessors/controller.py @@ -8,9 +8,9 @@ from conversion.export.profile import save_profile from conversion.leveling.solver.utils import compute_image_center from conversion.preprocess_impression.parameters import PreprocessingImpressionParams -from conversion.preprocess_impression.preprocess_impression import preprocess_impression_mark +from conversion.preprocess_impression.preprocess_impression import ImpressionParams, preprocess_impression_mark from conversion.preprocess_striation import PreprocessingStriationParams -from conversion.preprocess_striation.pipeline import preprocess_striation_mark +from conversion.preprocess_striation.pipeline import StriationParams, preprocess_striation_mark from conversion.resample import resample_mark from loguru import logger from mutations import CropToMask, GaussianRegressionFilter, LevelMap, Mask, Resample, Rotate @@ -22,7 +22,7 @@ from constants import LIGHT_SOURCES, OBSERVER from preprocessors.constants import PrepareMarkImpressionFiles, PrepareMarkStriationFiles from preprocessors.pipelines import preview_pipeline, surface_map_pipeline -from preprocessors.schemas import EditImage +from preprocessors.schemas import EditImage, PrepareMarkStriation def _scan_image_to_mark(mask: BinaryMask, bounding_box: BoundingBox | None, scan_image: ScanImage) -> ScanImage: @@ -87,7 +87,7 @@ def process_prepare_impression_mark( # noqa: PLR0913 mark_type: MarkType, mask: BinaryMask, bounding_box: BoundingBox | None, - preprocess_parameters: PreprocessingImpressionParams, + preprocess_parameters: ImpressionParams, working_dir: Path, ) -> None: """Prepare impression mark data.""" @@ -103,7 +103,7 @@ def process_prepare_striation_mark( # noqa: PLR0913 mark_type: MarkType, mask: BinaryMask, bounding_box: BoundingBox | None, - preprocess_parameters: PreprocessingStriationParams, + preprocess_parameters: StriationParams, working_dir: Path, ) -> None: """Prepare striation mark data.""" diff --git a/src/preprocessors/router.py b/src/preprocessors/router.py index 0aa48d22..42ad11da 100644 --- a/src/preprocessors/router.py +++ b/src/preprocessors/router.py @@ -1,5 +1,7 @@ from http import HTTPStatus +from conversion.preprocess_impression.preprocess_impression import ImpressionParams +from conversion.preprocess_striation.pipeline import StriationParams from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import RedirectResponse from loguru import logger @@ -120,13 +122,13 @@ async def prepare_mark_impression(params: PrepareMarkImpression = Depends()) -> ) except ArrayShapeMismatchError as e: raise HTTPException(status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=str(e)) - + impression_params = ImpressionParams(**params.model_dump()) process_prepare_impression_mark( scan_image=parsed_image, mark_type=params.mark_type, mask=parsed_mask, bounding_box=params.bounding_box, - preprocess_parameters=params.mark_parameters, + preprocess_parameters=impression_params, working_dir=vault.resource_path, ) logger.info(f"Generated files saved to {vault}") @@ -163,14 +165,14 @@ async def prepare_mark_striation(params: PrepareMarkStriation = Depends()) -> Pr ) except ArrayShapeMismatchError as e: raise HTTPException(status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail=str(e)) - + striation_params = StriationParams(**params.model_dump()) process_prepare_striation_mark( working_dir=vault.resource_path, scan_image=parsed_image, mark_type=params.mark_type, mask=parsed_mask, bounding_box=params.bounding_box, - preprocess_parameters=params.mark_parameters, + preprocess_parameters=striation_params, ) logger.info(f"Generated files saved to {vault}") return PrepareMarkResponseStriation.from_enum(enum=PrepareMarkStriationFiles, base_url=vault.access_url) From 965771ce537e44a5bd3d4d1a976e1b0d4245c6b5 Mon Sep 17 00:00:00 2001 From: raytesnel Date: Mon, 30 Mar 2026 07:42:56 +0200 Subject: [PATCH 5/9] update router.py docs --- src/preprocessors/router.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/preprocessors/router.py b/src/preprocessors/router.py index 42ad11da..5569e6e4 100644 --- a/src/preprocessors/router.py +++ b/src/preprocessors/router.py @@ -98,10 +98,8 @@ async def process_scan(upload_scan: UploadScan) -> ProcessedDataAccess: description=""" Applies user-defined masking and cropping to a scan, then performs mark-type-specific preprocessing (rotation, cropping, filtering) for impression marks. - Outputs two processed mark representations (.npz data and .json metadata) saved to the vault, returning URLs for file access. - The mask must have exactly the same shape (height × width) as the parsed scan image. """, responses={ @@ -141,10 +139,8 @@ async def prepare_mark_impression(params: PrepareMarkImpression = Depends()) -> description=""" Applies user-defined masking and cropping to a scan, then performs mark-type-specific preprocessing (rotation, cropping, filtering) for striation marks. - Outputs two processed mark representations (.npz data and .json metadata) saved to the vault, returning URLs for file access. - The mask must have exactly the same shape (height × width) as the parsed scan image. """, responses={ @@ -184,11 +180,8 @@ async def prepare_mark_striation(params: PrepareMarkStriation = Depends()) -> Pr description=""" Parse and validate a scan file (X3P format only) with the provided edit parameters (mask, crop, subsampling). Creates a new vault for storing future outputs. - The mask shape specified in `mask_parameters.shape` must exactly match the shape (height × width) of the parsed scan image. - - Note: Image generation is currently not implemented. """, responses={ HTTPStatus.BAD_REQUEST: {"description": "parse error"}, From e59c8ba94723a0535fd7a379af6c474d3a77d587 Mon Sep 17 00:00:00 2001 From: raytesnel Date: Mon, 30 Mar 2026 08:06:24 +0200 Subject: [PATCH 6/9] flatten params in striation and impression --- .../preprocess_impression.py | 17 +++++-- .../preprocess_striation/pipeline.py | 6 +-- .../test_preprocess_data.py | 12 +++-- .../test_striation_matlab.py | 1 - .../tests/conversion/test_impression.py | 3 +- .../conversion/test_impression_matlab.py | 11 ++-- .../test_preprocess_data_matlab_comparison.py | 11 ++-- src/preprocessors/controller.py | 4 +- .../test_apply_changes_on_scan_image.py | 15 +++++- tests/preprocessors/router/test_router.py | 35 +++++++------ .../preprocessors/schemas/test_edit_image.py | 50 +++++++++++-------- 11 files changed, 99 insertions(+), 66 deletions(-) diff --git a/packages/scratch-core/src/conversion/preprocess_impression/preprocess_impression.py b/packages/scratch-core/src/conversion/preprocess_impression/preprocess_impression.py index f9098a4b..4234ac89 100644 --- a/packages/scratch-core/src/conversion/preprocess_impression/preprocess_impression.py +++ b/packages/scratch-core/src/conversion/preprocess_impression/preprocess_impression.py @@ -4,8 +4,6 @@ (e.g., breech face impressions) through leveling, filtering, and resampling steps. """ -from dataclasses import asdict - from pydantic import BaseModel from container_models.base import DepthData @@ -17,7 +15,6 @@ from conversion.leveling import SurfaceTerms, level_map from conversion.mask import crop_to_mask from conversion.preprocess_impression.center import compute_center_local -from conversion.preprocess_impression.parameters import PreprocessingImpressionParams from conversion.preprocess_impression.resample import ( needs_resampling, resample, @@ -39,6 +36,18 @@ class ImpressionParams(BaseModel): highpass_regression_order: int = 2 lowpass_regression_order: int = 0 + @property + def surface_terms(self) -> SurfaceTerms: + """Convert leveling flags to SurfaceTerms.""" + terms = SurfaceTerms.NONE + if self.level_offset: + terms |= SurfaceTerms.OFFSET + if self.level_tilt: + terms |= SurfaceTerms.TILT_X | SurfaceTerms.TILT_Y + if self.level_2nd: + terms |= SurfaceTerms.ASTIG_45 | SurfaceTerms.DEFOCUS | SurfaceTerms.ASTIG_0 + return terms + def preprocess_impression_mark( mark: Mark, @@ -116,7 +125,7 @@ def preprocess_impression_mark( ) # Build output metadata - mark.meta_data.update(**asdict(params)) + mark.meta_data.update(**params.model_dump()) return mark_filtered, mark_leveled_final diff --git a/packages/scratch-core/src/conversion/preprocess_striation/pipeline.py b/packages/scratch-core/src/conversion/preprocess_striation/pipeline.py index 8627f006..6d7fbf18 100644 --- a/packages/scratch-core/src/conversion/preprocess_striation/pipeline.py +++ b/packages/scratch-core/src/conversion/preprocess_striation/pipeline.py @@ -6,8 +6,6 @@ - Fine rotation to align striations horizontally and profile extraction """ -from dataclasses import asdict - import numpy as np from pydantic import BaseModel @@ -18,7 +16,6 @@ apply_striation_preserving_filter_1d, cutoff_to_gaussian_sigma, ) -from conversion.preprocess_striation import PreprocessingStriationParams from conversion.preprocess_striation.alignment import fine_align_bullet_marks from conversion.preprocess_striation.shear import propagate_nan from conversion.profile_correlator import Profile @@ -33,6 +30,7 @@ class StriationParams(BaseModel): max_iter: int = 25 subsampling_factor: int = 1 + def preprocess_striation_mark( mark: Mark, params: StriationParams, @@ -99,7 +97,7 @@ def preprocess_striation_mark( # Build meta_data with mask and total_angle aligned_meta_data = { **mark.meta_data, - **asdict(params), + **params.model_dump(), "total_angle": total_angle, } diff --git a/packages/scratch-core/tests/conversion/preprocess_striation/test_preprocess_data.py b/packages/scratch-core/tests/conversion/preprocess_striation/test_preprocess_data.py index 44302c6f..317651fe 100644 --- a/packages/scratch-core/tests/conversion/preprocess_striation/test_preprocess_data.py +++ b/packages/scratch-core/tests/conversion/preprocess_striation/test_preprocess_data.py @@ -2,27 +2,29 @@ Tests for preprocess_striation.py and related filter functions. """ +from math import ceil + import numpy as np import pytest -from math import ceil from scipy.constants import micro from container_models.scan_image import ScanImage from conversion.data_formats import MarkType -from ..helper_functions import make_mark from conversion.filter import ( apply_striation_preserving_filter_1d, cutoff_to_gaussian_sigma, ) from conversion.filter.gaussian import _apply_nan_weighted_gaussian_1d from conversion.preprocess_striation import ( - PreprocessingStriationParams, apply_shape_noise_removal, fine_align_bullet_marks, preprocess_striation_mark, ) -from conversion.preprocess_striation.shear import shear_data_by_shifting_profiles from conversion.preprocess_striation.alignment import _detect_striation_angle +from conversion.preprocess_striation.pipeline import StriationParams +from conversion.preprocess_striation.shear import shear_data_by_shifting_profiles + +from ..helper_functions import make_mark def test_cutoff_to_gaussian_sigma(): @@ -336,7 +338,7 @@ def test_preprocess_striation_mark(): scale_y=micro, mark_type=MarkType.BULLET_LEA_STRIATION, ) - params = PreprocessingStriationParams( + params = StriationParams( highpass_cutoff=2e-3, lowpass_cutoff=2.5e-4, cut_borders_after_smoothing=False, diff --git a/packages/scratch-core/tests/conversion/preprocess_striation/test_striation_matlab.py b/packages/scratch-core/tests/conversion/preprocess_striation/test_striation_matlab.py index 91944548..5856d515 100755 --- a/packages/scratch-core/tests/conversion/preprocess_striation/test_striation_matlab.py +++ b/packages/scratch-core/tests/conversion/preprocess_striation/test_striation_matlab.py @@ -13,7 +13,6 @@ from container_models.base import DepthData from conversion.data_formats import MarkType from conversion.preprocess_striation import ( - PreprocessingStriationParams, preprocess_striation_mark, ) from conversion.preprocess_striation.pipeline import StriationParams diff --git a/packages/scratch-core/tests/conversion/test_impression.py b/packages/scratch-core/tests/conversion/test_impression.py index 2067e22b..a4f4ea8c 100644 --- a/packages/scratch-core/tests/conversion/test_impression.py +++ b/packages/scratch-core/tests/conversion/test_impression.py @@ -15,7 +15,6 @@ _points_are_collinear, compute_center_local, ) -from conversion.preprocess_impression.parameters import PreprocessingImpressionParams from conversion.preprocess_impression.preprocess_impression import ( ImpressionParams, preprocess_impression_mark, @@ -447,7 +446,7 @@ def test_output_has_correct_scale(self): mark_type=MarkType.FIRING_PIN_IMPRESSION, ) target_size = 2 * micro - params = ImpressionParams( + params = ImpressionParams( pixel_size=target_size, adjust_pixel_spacing=False, ) diff --git a/packages/scratch-core/tests/conversion/test_impression_matlab.py b/packages/scratch-core/tests/conversion/test_impression_matlab.py index 4d049891..eee213af 100644 --- a/packages/scratch-core/tests/conversion/test_impression_matlab.py +++ b/packages/scratch-core/tests/conversion/test_impression_matlab.py @@ -11,15 +11,16 @@ from container_models.base import FloatArray2D from conversion.data_formats import MarkType -from .helper_functions import make_mark from conversion.preprocess_impression.preprocess_impression import ( + ImpressionParams, preprocess_impression_mark, ) -from conversion.preprocess_impression.parameters import PreprocessingImpressionParams + from .helper_functions import ( _compute_correlation, - _crop_to_common_shape, _compute_difference_stats, + _crop_to_common_shape, + make_mark, ) @@ -32,7 +33,7 @@ class MatlabTestCase: input_data: FloatArray2D pixel_spacing: tuple[float, float] # Processing options - params: PreprocessingImpressionParams + params: ImpressionParams use_circle_center: bool # Expected output output_data: FloatArray2D @@ -89,7 +90,7 @@ def from_directory(cls, case_dir: Path) -> "MatlabTestCase": lowpass_cutoff = cutoff_val lowpass_order = int(f.get("n_order", 0)) - params = PreprocessingImpressionParams( + params = ImpressionParams( adjust_pixel_spacing=meta.get("adjust_pixel_spacing", False), level_offset=level_offset, level_tilt=level_tilt, diff --git a/packages/scratch-core/tests/conversion/test_preprocess_data_matlab_comparison.py b/packages/scratch-core/tests/conversion/test_preprocess_data_matlab_comparison.py index cf8c0282..33de816e 100644 --- a/packages/scratch-core/tests/conversion/test_preprocess_data_matlab_comparison.py +++ b/packages/scratch-core/tests/conversion/test_preprocess_data_matlab_comparison.py @@ -11,17 +11,18 @@ import pytest from scipy.constants import micro -from container_models.base import DepthData, BinaryMask, StriationProfile +from container_models.base import BinaryMask, DepthData, StriationProfile from conversion.data_formats import MarkType -from .helper_functions import make_mark from conversion.preprocess_striation import ( - PreprocessingStriationParams, preprocess_striation_mark, ) +from conversion.preprocess_striation.pipeline import StriationParams + from .helper_functions import ( _compute_correlation, - _crop_to_common_shape, _compute_difference_stats, + _crop_to_common_shape, + make_mark, ) @@ -170,7 +171,7 @@ def run_python_preprocessing( mark_type=mark_type, ) - params = PreprocessingStriationParams( + params = StriationParams( highpass_cutoff=test_case.cutoff_hi * micro, lowpass_cutoff=test_case.cutoff_lo * micro, use_mean=test_case.use_mean, diff --git a/src/preprocessors/controller.py b/src/preprocessors/controller.py index 0934fd52..525da39d 100644 --- a/src/preprocessors/controller.py +++ b/src/preprocessors/controller.py @@ -7,9 +7,7 @@ from conversion.export.mark import save_mark from conversion.export.profile import save_profile from conversion.leveling.solver.utils import compute_image_center -from conversion.preprocess_impression.parameters import PreprocessingImpressionParams from conversion.preprocess_impression.preprocess_impression import ImpressionParams, preprocess_impression_mark -from conversion.preprocess_striation import PreprocessingStriationParams from conversion.preprocess_striation.pipeline import StriationParams, preprocess_striation_mark from conversion.resample import resample_mark from loguru import logger @@ -22,7 +20,7 @@ from constants import LIGHT_SOURCES, OBSERVER from preprocessors.constants import PrepareMarkImpressionFiles, PrepareMarkStriationFiles from preprocessors.pipelines import preview_pipeline, surface_map_pipeline -from preprocessors.schemas import EditImage, PrepareMarkStriation +from preprocessors.schemas import EditImage def _scan_image_to_mark(mask: BinaryMask, bounding_box: BoundingBox | None, scan_image: ScanImage) -> ScanImage: diff --git a/tests/preprocessors/controller/test_apply_changes_on_scan_image.py b/tests/preprocessors/controller/test_apply_changes_on_scan_image.py index b97455c4..6b1dd40a 100644 --- a/tests/preprocessors/controller/test_apply_changes_on_scan_image.py +++ b/tests/preprocessors/controller/test_apply_changes_on_scan_image.py @@ -1,3 +1,4 @@ +import io from collections.abc import Callable from pathlib import Path @@ -5,6 +6,7 @@ import pytest from container_models.base import BinaryMask from container_models.scan_image import ScanImage +from fastapi import UploadFile from parsers import convert_to_x3p, save_x3p from scipy.constants import micro from utils.constants import RegressionOrder @@ -35,6 +37,7 @@ def resample_twice_bigger( save_x3p(output_path=scan_file, x3p=convert_to_x3p(scan_image)) mask = np.ones(shape=(2, 3), dtype=np.bool) + mask_file = UploadFile(file=io.BytesIO(mask.tobytes()), filename="mask.bin") params = EditImage( project_name="test", @@ -44,6 +47,8 @@ def resample_twice_bigger( terms=SurfaceOptions.PLANE, regression_order=RegressionOrder.GAUSSIAN_WEIGHTED_AVERAGE, crop=False, + mask_data=mask_file, + mask_is_bitpacked=False, ) def assertions(result: ScanImage) -> None: @@ -66,6 +71,7 @@ def mask_middle_pixel(scan_image: ScanImage, tmp_path: Path) -> tuple[EditImage, ], dtype=np.bool, ) + mask_file = UploadFile(file=io.BytesIO(mask.tobytes()), filename="mask.bin") params = EditImage( project_name="test", @@ -75,6 +81,8 @@ def mask_middle_pixel(scan_image: ScanImage, tmp_path: Path) -> tuple[EditImage, terms=SurfaceOptions.PLANE, regression_order=RegressionOrder.GAUSSIAN_WEIGHTED_AVERAGE, crop=False, + mask_data=mask_file, + mask_is_bitpacked=False, ) def assertions(result: ScanImage): @@ -96,6 +104,7 @@ def crop_to_middle_pixel(scan_image: ScanImage, tmp_path: Path) -> tuple[EditIma ], dtype=np.bool, ) + mask_file = UploadFile(file=io.BytesIO(mask.tobytes()), filename="mask.bin") params = EditImage( project_name="test", @@ -105,6 +114,8 @@ def crop_to_middle_pixel(scan_image: ScanImage, tmp_path: Path) -> tuple[EditIma terms=SurfaceOptions.PLANE, regression_order=RegressionOrder.GAUSSIAN_WEIGHTED_AVERAGE, crop=True, + mask_data=mask_file, + mask_is_bitpacked=False, ) def assertions(result: ScanImage): @@ -126,7 +137,7 @@ def crop_to_resized_image(scan_image: ScanImage, tmp_path: Path) -> tuple[EditIm ], dtype=np.bool, ) - + mask_file = UploadFile(file=io.BytesIO(mask.tobytes()), filename="mask.bin") params = EditImage( project_name="test", scan_file=scan_file, @@ -135,6 +146,8 @@ def crop_to_resized_image(scan_image: ScanImage, tmp_path: Path) -> tuple[EditIm terms=SurfaceOptions.PLANE, regression_order=RegressionOrder.GAUSSIAN_WEIGHTED_AVERAGE, crop=True, + mask_data=mask_file, + mask_is_bitpacked=False, ) def assertions(result: ScanImage): diff --git a/tests/preprocessors/router/test_router.py b/tests/preprocessors/router/test_router.py index 927762c7..aa71ae3e 100644 --- a/tests/preprocessors/router/test_router.py +++ b/tests/preprocessors/router/test_router.py @@ -1,10 +1,13 @@ from http import HTTPStatus +from io import BytesIO from pathlib import Path import numpy as np import pytest from container_models.base import BinaryMask from conversion.data_formats import MarkType +from conversion.preprocess_impression.preprocess_impression import ImpressionParams +from conversion.preprocess_striation.pipeline import StriationParams from fastapi.testclient import TestClient from httpx import Response from pydantic import HttpUrl @@ -21,8 +24,6 @@ PrepareMarkResponseImpression, PrepareMarkResponseStriation, PrepareMarkStriation, - PreprocessingImpressionParams, - PreprocessingStriationParams, ) from settings import get_settings @@ -30,9 +31,13 @@ def send_post_request_with_mask(client: TestClient, endpoint: str, params: dict, mask: BinaryMask) -> Response: return client.post( f"{get_settings().base_url}/{RoutePrefix.PREPROCESSOR}/{endpoint}", - params=params, + params={k: v for k, v in params.items() if v is not None}, files={ - "mask_data": ("mask.bin", mask.tobytes(order="C"), "application/octet-stream"), + "mask_data": ( + "mask.bin", + BytesIO(mask.tobytes(order="C")), + "application/octet-stream", + ) }, timeout=5, ) @@ -57,7 +62,7 @@ def test_pre_processors_placeholder(client: TestClient) -> None: PreprocessorEndpoint.PREPARE_MARK_STRIATION, PrepareMarkStriation, PrepareMarkResponseStriation, - PreprocessingStriationParams, + StriationParams, MarkType.APERTURE_SHEAR_STRIATION, [ "preview_image", @@ -75,7 +80,7 @@ def test_pre_processors_placeholder(client: TestClient) -> None: PreprocessorEndpoint.PREPARE_MARK_IMPRESSION, PrepareMarkImpression, PrepareMarkResponseImpression, - PreprocessingImpressionParams, + ImpressionParams, MarkType.CHAMBER_IMPRESSION, [ "preview_image", @@ -109,7 +114,7 @@ def get_schema_for_endpoint( self, schema: type[PrepareMarkImpression | PrepareMarkStriation], mark_type: str, - mark_parameters: type[PreprocessingStriationParams | PreprocessingImpressionParams], + mark_parameters: type[StriationParams | ImpressionParams], ): """Generate the schema payload for the prepare-mark endpoint.""" return schema.model_construct( @@ -125,7 +130,7 @@ def test_prepare_mark_endpoint_returns_urls( # noqa: PLR0913 endpoint: PreprocessorEndpoint, schema: type[PrepareMarkImpression | PrepareMarkStriation], response_schema: type[PrepareMarkResponseImpression | PrepareMarkResponseStriation], - mark_parameters: type[PreprocessingStriationParams | PreprocessingImpressionParams], + mark_parameters: type[StriationParams | ImpressionParams], mark_type: str, mask: BinaryMask, expected_keys: list[str], @@ -156,7 +161,7 @@ def test_prepare_mark_endpoint_has_made_files_in_vault( # noqa: PLR0913 schema: type[PrepareMarkImpression | PrepareMarkStriation], response_schema: type[PrepareMarkResponseImpression | PrepareMarkResponseStriation], endpoint: PreprocessorEndpoint, - mark_parameters: PreprocessingStriationParams | PreprocessingImpressionParams, + mark_parameters: StriationParams | ImpressionParams, mask: BinaryMask, mark_type: str, expected_keys: list[str], @@ -185,7 +190,7 @@ def test_prepare_mark_endpoint_response_url_matches_folder_location( # noqa: PL schema: type[PrepareMarkImpression | PrepareMarkStriation], response_schema: type[PrepareMarkResponseImpression | PrepareMarkResponseStriation], endpoint: PreprocessorEndpoint, - mark_parameters: PreprocessingStriationParams | PreprocessingImpressionParams, + mark_parameters: StriationParams | ImpressionParams, mask: BinaryMask, mark_type: str, expected_keys: list[str], @@ -218,14 +223,14 @@ def test_prepare_mark_endpoint_response_url_matches_folder_location( # noqa: PL pytest.param( PreprocessorEndpoint.PREPARE_MARK_STRIATION, PrepareMarkStriation, - PreprocessingStriationParams, + StriationParams, MarkType.APERTURE_SHEAR_STRIATION, id="striation mark", ), pytest.param( PreprocessorEndpoint.PREPARE_MARK_IMPRESSION, PrepareMarkImpression, - PreprocessingImpressionParams, + ImpressionParams, MarkType.CHAMBER_IMPRESSION, id="impression mark", ), @@ -238,17 +243,15 @@ def test_prepare_mark_returns_422_on_mask_shape_mismatch( # noqa: PLR0913 monkeypatch: pytest.MonkeyPatch, endpoint: PreprocessorEndpoint, schema: type[PrepareMarkImpression | PrepareMarkStriation], - mark_parameters: type[PreprocessingStriationParams | PreprocessingImpressionParams], + mark_parameters: type[StriationParams | ImpressionParams], mark_type: MarkType, ) -> None: """Test that a 422 is returned when the mask shape does not match the scan image shape.""" wrong_mask = np.zeros(shape=(2, 2), dtype=np.bool_) # 2x2, won't match the scan shape - - payload = schema( + payload = schema.model_construct( project_name="test_project", mark_type=mark_type, scan_file=scan_directory / "circle.x3p", - mark_parameters=mark_parameters(), # type: ignore bounding_box_list=[], ).model_dump(mode="json") diff --git a/tests/preprocessors/schemas/test_edit_image.py b/tests/preprocessors/schemas/test_edit_image.py index a592bd67..2d22695a 100644 --- a/tests/preprocessors/schemas/test_edit_image.py +++ b/tests/preprocessors/schemas/test_edit_image.py @@ -1,9 +1,12 @@ from collections.abc import Callable +from io import BytesIO from itertools import chain from pathlib import Path from typing import Any, Final +import numpy as np import pytest +from fastapi import UploadFile from hypothesis import given from hypothesis import strategies as st from pydantic import ValidationError @@ -24,6 +27,16 @@ def get_error_fields(exc_info, typ: str) -> tuple[str, ...]: class TestEditImage: """Tests for EditImage request model.""" + DEFAULT_WIDTH_SCAN_CIRCLE = 259 + DEFAULT_HEIGHT_SCAN_CIRCLE = 259 + + def _empty_mask(self, shape_width: int, shape_height: int) -> UploadFile: + mask = np.ones( + (shape_height, shape_width), + dtype=np.bool, + ) + return UploadFile(file=BytesIO(mask.tobytes()), filename="mask.bin") + def test_should_reject_non_x3p_file( self, scan_directory: Path, edit_image_parameter: Callable[..., EditImage] ) -> None: @@ -33,7 +46,10 @@ def test_should_reject_non_x3p_file( # Act & Assert with pytest.raises(ValidationError, match="Unsupported extension") as exc_info: - edit_image_parameter(scan_file=al3d_file) + edit_image_parameter( + scan_file=al3d_file, + mask_data=self._empty_mask(self.DEFAULT_HEIGHT_SCAN_CIRCLE, self.DEFAULT_WIDTH_SCAN_CIRCLE), + ) # Assert errors = exc_info.value.errors() @@ -55,21 +71,6 @@ def test_should_reject_non_existent_file( errors = exc_info.value.errors() assert any("scan_file" in error["loc"] for error in errors) - def test_should_create_with_all_defaults(self, edit_image_parameter: Callable[..., EditImage]) -> None: - """Test that EditImage can be created with default values when mask and cutoff_length are provided.""" - # Arrange - - # Act - params = edit_image_parameter() - - # Assert - assert params.resampling_factor == DEFAULT_RESAMPLING_FACTOR - assert params.terms == SurfaceOptions.NONE - assert params.regression_order == RegressionOrder.GAUSSIAN_WEIGHTED_AVERAGE - assert params.cutoff_length == CUTOFF_LENGTH - assert params.crop is False - assert params.project_name is None - @pytest.mark.parametrize( "kwargs", [ @@ -85,8 +86,11 @@ def test_should_accept_valid_field_values( self, kwargs: dict[str, Any], edit_image_parameter: Callable[..., EditImage] ) -> None: """Test that enum and mask field values are accepted.""" + non_optional_fields = { + "mask_data": self._empty_mask(self.DEFAULT_HEIGHT_SCAN_CIRCLE, self.DEFAULT_WIDTH_SCAN_CIRCLE) + } # Act - params = edit_image_parameter(**kwargs) + params = edit_image_parameter(**kwargs, **non_optional_fields) # Assert assert all(getattr(params, field) == value for field, value in kwargs.items()) @@ -97,7 +101,10 @@ def test_should_accept_positive_cutoff_length( ) -> None: """Test that positive step sizes are accepted.""" # Act - params = edit_image_parameter(cutoff_length=valid_value) + params = edit_image_parameter( + cutoff_length=valid_value, + mask_data=self._empty_mask(self.DEFAULT_HEIGHT_SCAN_CIRCLE, self.DEFAULT_WIDTH_SCAN_CIRCLE), + ) # Assert assert params.cutoff_length == valid_value @@ -108,7 +115,10 @@ def test_should_accept_positive_resampling_factor( ) -> None: """Test that positive step sizes are accepted.""" # Act - params = edit_image_parameter(resampling_factor=valid_value) + params = edit_image_parameter( + resampling_factor=valid_value, + mask_data=self._empty_mask(self.DEFAULT_HEIGHT_SCAN_CIRCLE, self.DEFAULT_WIDTH_SCAN_CIRCLE), + ) # Assert assert params.resampling_factor == valid_value @@ -134,4 +144,4 @@ def test_should_reject_when_required_fields_not_provided(self) -> None: EditImage() # type: ignore # Assert - assert get_error_fields(exc_info, "missing") == ("scan_file", "cutoff_length", "terms") + assert get_error_fields(exc_info, "missing") == ("scan_file", "cutoff_length", "terms", "mask_data") From 2836e5d1aa7850f285408fefd659456e6ec9dad0 Mon Sep 17 00:00:00 2001 From: raytesnel Date: Tue, 31 Mar 2026 10:37:37 +0200 Subject: [PATCH 7/9] update swagger docs with description --- src/preprocessors/router.py | 12 ++++---- src/preprocessors/schemas.py | 12 ++++++-- src/schemas.py | 14 ++++++++++ tests/test_generate_description.py | 45 ++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 8 deletions(-) create mode 100644 tests/test_generate_description.py diff --git a/src/preprocessors/router.py b/src/preprocessors/router.py index 5569e6e4..f23b5ac1 100644 --- a/src/preprocessors/router.py +++ b/src/preprocessors/router.py @@ -14,6 +14,7 @@ ) from file_services import create_vault from preprocessors.controller import edit_scan_image, process_prepare_impression_mark, process_prepare_striation_mark +from schemas import generate_description from .constants import GeneratedImageFiles, PrepareMarkImpressionFiles, PrepareMarkStriationFiles, ProcessFiles from .exceptions import ArrayShapeMismatchError @@ -95,12 +96,12 @@ async def process_scan(upload_scan: UploadScan) -> ProcessedDataAccess: @preprocessor_route.post( path=f"/{PreprocessorEndpoint.PREPARE_MARK_IMPRESSION}", summary="Preprocess a scan into analysis-ready impression mark files.", - description=""" + description=f""" Applies user-defined masking and cropping to a scan, then performs mark-type-specific preprocessing (rotation, cropping, filtering) for impression marks. Outputs two processed mark representations (.npz data and .json metadata) saved to the vault, returning URLs for file access. - The mask must have exactly the same shape (height × width) as the parsed scan image. + {generate_description(PrepareMarkImpression)} """, responses={ HTTPStatus.UNPROCESSABLE_ENTITY: {"description": "mask shape does not match image shape"}, @@ -136,12 +137,12 @@ async def prepare_mark_impression(params: PrepareMarkImpression = Depends()) -> @preprocessor_route.post( path=f"/{PreprocessorEndpoint.PREPARE_MARK_STRIATION}", summary="Preprocess a scan into analysis-ready striation mark files.", - description=""" + description=f""" Applies user-defined masking and cropping to a scan, then performs mark-type-specific preprocessing (rotation, cropping, filtering) for striation marks. Outputs two processed mark representations (.npz data and .json metadata) saved to the vault, returning URLs for file access. - The mask must have exactly the same shape (height × width) as the parsed scan image. + {generate_description(PrepareMarkStriation)} """, responses={ HTTPStatus.UNPROCESSABLE_ENTITY: {"description": "mask shape does not match image shape"}, @@ -177,11 +178,12 @@ async def prepare_mark_striation(params: PrepareMarkStriation = Depends()) -> Pr @preprocessor_route.post( path=f"/{PreprocessorEndpoint.EDIT_SCAN}", summary="Validate and parse a scan file with edit parameters.", - description=""" + description=f""" Parse and validate a scan file (X3P format only) with the provided edit parameters (mask, crop, subsampling). Creates a new vault for storing future outputs. The mask shape specified in `mask_parameters.shape` must exactly match the shape (height × width) of the parsed scan image. + {generate_description(EditImage)} """, responses={ HTTPStatus.BAD_REQUEST: {"description": "parse error"}, diff --git a/src/preprocessors/schemas.py b/src/preprocessors/schemas.py index 677c857e..39d57036 100644 --- a/src/preprocessors/schemas.py +++ b/src/preprocessors/schemas.py @@ -98,7 +98,9 @@ class PrepareMarkStriation(PrepareMarkBase): angle_accuracy: float = 0.1 max_iter: int = 25 subsampling_factor: int = 1 - mask_data: UploadFile = File() + mask_data: UploadFile = File( + ..., description="Mask given as binary data. The shape of the mask needs to be the same as scan_image." + ) @field_validator("mark_type") @classmethod @@ -120,7 +122,9 @@ class PrepareMarkImpression(PrepareMarkBase): lowpass_cutoff: float | None = 5.0e-6 highpass_regression_order: int = 2 lowpass_regression_order: int = 0 - mask_data: UploadFile = File() + mask_data: UploadFile = File( + ..., description="Mask given as binary data. The shape of the mask needs to be the same as scan_image." + ) @field_validator("mark_type") @classmethod @@ -164,7 +168,9 @@ class EditImage(BaseParameters): 'The expected bit-order for bit-packed arrays is "little".', examples=[True, False], ) - mask_data: UploadFile = File() + mask_data: UploadFile = File( + ..., description="Mask given as binary data. The shape of the mask needs to be the same as scan_image." + ) @model_validator(mode="after") def check_file_is_x3p(self): diff --git a/src/schemas.py b/src/schemas.py index 41a38c54..2da54cb4 100644 --- a/src/schemas.py +++ b/src/schemas.py @@ -17,3 +17,17 @@ def from_enum( ) -> C: """Initiate the Response model with the given files from the enum.""" return cls(**{file.name: HttpUrl(f"{base_url}/{file.value}") for file in enum}) + + +def generate_description(model: type[BaseModel]) -> str: + """Generate a description field for in swagger docs for development purpose.""" + lines = ["\n\n---\n\nExpected form data:"] + + for name, field in model.model_fields.items(): + required = "required" if field.is_required() else "optional" + default = f" (default: {field.default})" if field.default is not None and not field.is_required() else "" + desc = field.description or "" + + lines.append(f"- `{name}` ({required}){default}\n {desc}") + + return "\n".join(lines) diff --git a/tests/test_generate_description.py b/tests/test_generate_description.py new file mode 100644 index 00000000..a360912d --- /dev/null +++ b/tests/test_generate_description.py @@ -0,0 +1,45 @@ +from pydantic import BaseModel, Field + +from schemas import generate_description + + +class UserModel(BaseModel): + name: str = Field(..., description="User full name") + age: int = Field(18, description="User age") + nickname: str | None = Field(None, description="Optional nickname") + + +def test_generate_description_distinguishes_required_and_optional_fields(): + # Arrange + expected_field_requirements = [ + "`name` (required)", + "`age` (optional)", + "`nickname` (optional)", + ] + # Act + result = generate_description(UserModel) + # Assert + for field_requirement in expected_field_requirements: + assert field_requirement in result, "Optional fields should be marked as optional" + + +def test_generate_description_has_formatted_field(): + # Arrange + expected_field = "- `name` (required)\n User full name" + # Act + result = generate_description(UserModel) + # Assert + assert expected_field in result, "Field should be formatted as required" + + +def test_generate_description_has_beginning(): + # Arrange + class SmallModel(BaseModel): + name: str = Field(..., description="User full name") + + # Act + result = generate_description(SmallModel) + # Assert + assert "\n\n---\n\nExpected form data:\n- `name` (required)\n User full name" == result, ( + "It should start with expected header" + ) From 3302a72cd5f03acbc74875ab825f8e7d104fd5c2 Mon Sep 17 00:00:00 2001 From: raytesnel Date: Tue, 31 Mar 2026 10:39:30 +0200 Subject: [PATCH 8/9] remove project tag ( in swagger there was an empty pattern as description `pattern:`) --- src/models.py | 5 ++--- src/preprocessors/schemas.py | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/models.py b/src/models.py index 0f813f0e..3174ce78 100644 --- a/src/models.py +++ b/src/models.py @@ -7,7 +7,7 @@ from typing import Annotated from uuid import uuid4 -from pydantic import UUID4, AfterValidator, BaseModel, ConfigDict, Field, FilePath, StringConstraints +from pydantic import UUID4, AfterValidator, BaseModel, ConfigDict, Field, FilePath from constants import RoutePrefix from settings import get_settings @@ -83,7 +83,6 @@ def validate_relative_path(filepath: Path) -> Path: return filepath -type ProjectTag = Annotated[str, StringConstraints(pattern=r"")] type ScanFile = Annotated[ FilePath, AfterValidator(lambda filepath: validate_file_extension(filepath, SupportedScanExtension)), @@ -119,7 +118,7 @@ class DirectoryAccess(BaseModelConfig): "Auto-generated to ensure no collisions with existing directories." ), ) - tag: ProjectTag = Field( + tag: str = Field( ..., description=( "Project tag for directory organization. " diff --git a/src/preprocessors/schemas.py b/src/preprocessors/schemas.py index 39d57036..fb122dd3 100644 --- a/src/preprocessors/schemas.py +++ b/src/preprocessors/schemas.py @@ -17,7 +17,6 @@ from models import ( BaseModelConfig, - ProjectTag, ScanFile, SupportedScanExtension, ) @@ -28,7 +27,7 @@ class BaseParameters(BaseModelConfig): """Base parameters for preprocessor operations including scan file.""" - project_name: ProjectTag | None = Field( + project_name: str | None = Field( None, description=( "Optional project identifier for organizing edited scans. " From 79eb18415f7e60c0c67ec264fe4403913690ada6 Mon Sep 17 00:00:00 2001 From: raytesnel Date: Tue, 31 Mar 2026 10:44:25 +0200 Subject: [PATCH 9/9] update contract tests --- tests/test_contracts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 28572603..2306ca92 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,4 +1,3 @@ -import json from enum import StrEnum from http import HTTPStatus from pathlib import Path @@ -42,7 +41,7 @@ class EndpointContractInterface(BaseModel): def send_post_request_with_mask(endpoint: str, params: dict, mask_raw: bytes) -> Response: return requests.post( f"{get_settings().base_url}/{RoutePrefix.PREPROCESSOR}/{endpoint}", - data={"params": json.dumps(params, default=str)}, + params=params, files={"mask_data": ("mask.bin", mask_raw, "application/octet-stream")}, timeout=5, )