fix(wsi): Add multi-file pyramid + WSI selection for DICOM files (#270)

* Add `select_dicom_files` which filters out non-WSI files using `SOPClassUID` and selects onlu the highest resolution image from multi-file pyramids to be processed
* Integrate into `wsi` service to handle files with `.dcm` extension
* Refactor `_scan_files` in `PydicomHandler` to reduce complexity, and add WSI /pyramid filtering as an option
* Update docs accordingly, including mention of highdicom now being optional

Author: blanca-pablos

src/aignostics/CLAUDE.md

@@ -246,7 +246,7 @@ aignostics application run submit --application-id heta --files slide.svs
 aignostics dataset download --collection-id TCGA-LUAD --output-dir ./data
 
 # Get WSI info
-aignostics wsi info slide.svs
+aignostics wsi inspect slide.svs
 ```
 
 ## GUI Launch

src/aignostics/application/_service.py

@@ -14,9 +14,7 @@
 from loguru import logger
 
 from aignostics.bucket import Service as BucketService
-from aignostics.constants import (
-    TEST_APP_APPLICATION_ID,
-)
+from aignostics.constants import TEST_APP_APPLICATION_ID
 from aignostics.platform import (
     LIST_APPLICATION_RUNS_MAX_PAGE_SIZE,
     ApiException,
@@ -32,9 +30,7 @@
     RunOutput,
     RunState,
 )
-from aignostics.platform import (
-    Service as PlatformService,
-)
+from aignostics.platform import Service as PlatformService
 from aignostics.utils import BaseService, Health, sanitize_path_component
 from aignostics.wsi import Service as WSIService
 
@@ -324,60 +320,60 @@ def generate_metadata_from_source_directory(  # noqa: PLR0913, PLR0917
         """Generate metadata from the source directory.
 
         Steps:
-        1. Recursively files ending with supported extensions in the source directory
-        2. Creates a dict with the following columns
+        1. Recursively scans files ending with supported extensions in the source directory
+        2. For DICOM files (.dcm), filters out auxiliary and redundant files
+        3. Creates a dict for each file with the following fields:
             - external_id (str): The external_id of the file, by default equivalent to the absolute file name
             - source (str): The absolute filename
-            - checksum_base64_crc32c (str): The CRC32C checksum of the file constructed, base64 encoded
+            - checksum_base64_crc32c (str): The CRC32C checksum of the file, base64 encoded
             - resolution_mpp (float): The microns per pixel, inspecting the base layer
-            - height_px: The height of the image in pixels, inspecting the base layer
-            - width_px: The width of the image in pixels, inspecting the base layer
-            - Further attributes depending on the application and it's version
-        3. Applies the optional mappings to fill in additional metadata fields in the dict.
+            - height_px (int): The height of the image in pixels, inspecting the base layer
+            - width_px (int): The width of the image in pixels, inspecting the base layer
+            - Further attributes depending on the application and its version
+        4. Applies the optional mappings to fill in additional metadata fields in the dict
 
         Args:
-            source_directory (Path): The source directory to generate metadata from.
-            application_id (str): The ID of the application.
-            application_version (str|None): The version of the application (semver).
-                If not given latest version is used.
-            with_gui_metadata (bool): If True, include additional metadata for GUI.
-            mappings (list[str]): Mappings of the form '<regexp>:<key>=<value>,<key>=<value>,...'.
+            source_directory: The source directory to generate metadata from.
+            application_id: The ID of the application.
+            application_version: The version of the application (semver).
+                If not given, latest version is used.
+            with_gui_metadata: If True, include additional metadata for GUI display.
+            mappings: Mappings of the form '<regexp>:<key>=<value>,<key>=<value>,...'.
                 The regular expression is matched against the external_id attribute of the entry.
                 The key/value pairs are applied to the entry if the pattern matches.
-            with_extra_metadata (bool): If True, include extra metadata from the WSIService.
+            with_extra_metadata: If True, include extra metadata from the WSIService.
 
         Returns:
-            dict[str, Any]: The generated metadata.
+            List of metadata dictionaries, one per processable file found.
 
         Raises:
-            Exception: If the metadata cannot be generated.
-
-        Raises:
-            NotFoundError: If the application version with the given ID is not found.
-            ValueError: If
-                the source directory does not exist
-                or is not a directory.
+            NotFoundException: If the application version with the given ID is not found.
+            ValueError: If the source directory does not exist or is not a directory.
             RuntimeError: If the metadata generation fails unexpectedly.
         """
         logger.trace("Generating metadata from source directory: {}", source_directory)
 
         # TODO(Helmut): Use it
         _ = Service().application_version(application_id, application_version)
 
-        metadata = []
+        metadata: list[dict[str, Any]] = []
+        wsi_service = WSIService()
 
         try:
             extensions = get_supported_extensions_for_application(application_id)
             for extension in extensions:
-                for file_path in source_directory.glob(f"**/*{extension}"):
+                files_to_process = wsi_service.get_wsi_files_to_process(source_directory, extension)
+
+                for file_path in files_to_process:
                     # Generate CRC32C checksum with crc32c and encode as base64
                     hash_sum = crc32c.CRC32CHash()
                     with file_path.open("rb") as f:
                         while chunk := f.read(1024):
                             hash_sum.update(chunk)
                     checksum = str(base64.b64encode(hash_sum.digest()), "UTF-8")
+
                     try:
-                        image_metadata = WSIService().get_metadata(file_path)
+                        image_metadata = wsi_service.get_metadata(file_path)
                         width = image_metadata["dimensions"]["width"]
                         height = image_metadata["dimensions"]["height"]
                         mpp = image_metadata["resolution"]["mpp_x"]

src/aignostics/wsi/CLAUDE.md

@@ -19,8 +19,8 @@ The WSI module provides comprehensive support for medical imaging files, particu
 **CLI Commands (`_cli.py`):**
 
 - `wsi inspect` - Display WSI file metadata and properties
-- `wsi dicom-inspect` - Inspect DICOM-specific metadata
-- `wsi dicom-geojson-import` - Import GeoJSON annotations to DICOM
+- `wsi dicom inspect` - Inspect DICOM-specific metadata
+- `wsi dicom geojson_import` - Import GeoJSON annotations to DICOM
 
 **GUI Component (`_gui.py`):**
 
@@ -210,6 +210,138 @@ def get_tile(
     return tile
 ```
 
+
+### DICOM WSI File Filtering
+
+**Multi-File DICOM Pyramid Selection (`_utils.select_dicom_files()`):**
+
+The WSI module automatically handles multi-file DICOM pyramids (whole slide images stored across multiple DICOM instances) by selecting only the highest resolution file from each pyramid. This prevents redundant processing since OpenSlide can automatically find related pyramid files in the same directory.
+
+**Implementation Location:**
+
+The DICOM file selection logic is implemented in `_utils.py` as `select_dicom_files()`. This function **only depends on pydicom** (not highdicom), making it compatible with Python 3.14+ where highdicom is not available.
+
+**Service Integration (`Service.get_wsi_files_to_process()`):**
+```python
+from aignostics.wsi import Service
+from pathlib import Path
+
+# Get filtered DICOM files
+files = Service.get_wsi_files_to_process(
+    path=Path("/data/dicoms"),
+    extension=".dcm"
+)
+# Returns only highest resolution WSI files
+
+# For non-DICOM formats, returns all files
+tiff_files = Service.get_wsi_files_to_process(
+    path=Path("/data/slides"),
+    extension=".tiff"
+)
+# Returns all .tiff files (no filtering)
+```
+
+**Direct Usage (Advanced):**
+```python
+from aignostics.wsi._utils import select_dicom_files
+from pathlib import Path
+
+# Directly filter DICOM files (used internally by Service)
+dicom_files = select_dicom_files(Path("/data/dicoms"))
+# Returns only highest resolution WSI files
+```
+
+**Filtering Strategy:**
+```python
+def select_dicom_files(path: Path) -> list[Path]:
+    """Select WSI files only, excluding auxiliary and redundant files.
+    
+    Filtering Strategy:
+    1. SOPClassUID filtering - Only process VL Whole Slide Microscopy Image Storage
+       - Include: 1.2.840.10008.5.1.4.1.1.77.1.6 (VL WSI)
+       - Exclude: 1.2.840.10008.5.1.4.1.1.66.4 (Segmentation Storage)
+       - Exclude: Other non-WSI DICOM types
+    
+    2. ImageType filtering - Exclude auxiliary images
+       - Keep: VOLUME images only
+       - Exclude: THUMBNAIL, LABEL, OVERVIEW, MACRO, ANNOTATION, LOCALIZER
+    
+    3. PyramidUID grouping - Group multi-file pyramids
+       - Files with same PyramidUID are part of one logical WSI
+       - Files without PyramidUID are treated as standalone WSIs
+    
+    4. Resolution selection - Keep highest resolution per pyramid
+       - Based on TotalPixelMatrixRows × TotalPixelMatrixColumns
+       - Excludes all lower resolution levels
+    
+    Reference: https://dicom.nema.org/medical/dicom/current/output/chtml/part03/chapter_7.html
+    """
+```
+
+**Key Behaviors:**
+
+- **SOPClassUID validation**: Only processes VL Whole Slide Microscopy Image Storage files (1.2.840.10008.5.1.4.1.1.77.1.6)
+- **Non-WSI exclusion**: Automatically excludes segmentations (1.2.840.10008.5.1.4.1.1.66.4), annotations, and other DICOM object types
+- **ImageType filtering**: Excludes THUMBNAIL, LABEL, OVERVIEW, MACRO, ANNOTATION, and LOCALIZER image types
+- **PyramidUID grouping**: Groups files by PyramidUID (DICOM tag identifying multi-resolution pyramids)
+- **Resolution selection**: For each pyramid, keeps only the file with largest TotalPixelMatrixRows × TotalPixelMatrixColumns
+- **Standalone handling**: Files without PyramidUID are treated as standalone WSI images and preserved
+- **Graceful degradation**: Files with missing attributes are logged and treated as standalone (not excluded)
+- **Debug logging**: Excluded files are logged at DEBUG level with pyramid/exclusion details
+
+**DICOM WSI Structure:**
+
+In the DICOM Whole Slide Imaging standard:
+- **PyramidUID**: Uniquely identifies a single multi-resolution pyramid that may span multiple files
+- **SeriesInstanceUID**: Groups related images (may include multiple pyramids, thumbnails, labels)
+- **TotalPixelMatrixRows/Columns**: Represents full image dimensions at the highest resolution level
+
+**Example Scenario:**
+```
+Input Directory:
+├── pyramid_level_0.dcm    (10000×10000 px, PyramidUID: ABC123) ← KEPT
+├── pyramid_level_1.dcm    (5000×5000 px,   PyramidUID: ABC123) ← EXCLUDED
+├── pyramid_level_2.dcm    (2500×2500 px,   PyramidUID: ABC123) ← EXCLUDED
+├── thumbnail.dcm          (256×256 px,     PyramidUID: ABC123, ImageType: THUMBNAIL) ← EXCLUDED
+├── segmentation.dcm       (10000×10000 px, SOPClassUID: Segmentation) ← EXCLUDED
+└── standalone.dcm         (8000×8000 px,   No PyramidUID) ← KEPT
+
+Result: Only pyramid_level_0.dcm and standalone.dcm are processed
+```
+
+**Error Handling:**
+
+- Files with missing SOPClassUID are logged as warnings and excluded (malformed DICOM)
+- Files with PyramidUID but missing TotalPixelMatrix* attributes are treated as standalone
+- Files that cannot be read by pydicom are logged at DEBUG level and skipped
+- AttributeError and general exceptions are caught to prevent processing pipeline failure
+
+**Integration with Application Module:**
+
+The Application module uses this filtering automatically when generating metadata:
+```python
+# In Application Service
+from aignostics.wsi import Service as WSIService
+
+# Filtering happens automatically for DICOM files
+files = WSIService.get_wsi_files_to_process(source_directory, ".dcm")
+for file_path in files:
+    # Only highest resolution WSI files are processed
+    metadata = WSIService.get_metadata(file_path)
+```
+
+**Module Architecture:**
+
+The DICOM file selection functionality is organized as follows:
+- **`_utils.py`**: Contains `select_dicom_files()` and `_find_highest_resolution_files()` helper
+  - Only depends on `pydicom`, `pathlib`, `collections.defaultdict`, and `loguru`
+  - Compatible with Python 3.14+ (no highdicom dependency)
+- **`_service.py`**: Uses `select_dicom_files()` in `get_wsi_files_to_process()`
+- **`_pydicom_handler.py`**: Uses `select_dicom_files()` for metadata extraction with `wsi_only=True`
+  - This module still requires highdicom for annotation/measurement features
+  - Only the CLI commands that need highdicom (geojson import, detailed inspection) use PydicomHandler
+
+
 ## Usage Patterns
 
 ### Basic WSI Operations
@@ -264,13 +396,10 @@ for wsi in wsi_files:
 aignostics wsi inspect slide.svs
 
 # Inspect DICOM metadata
-aignostics wsi dicom-inspect scan.dcm
+aignostics wsi dicom inspect scan.dcm
 
 # Import GeoJSON annotations
-aignostics wsi dicom-geojson-import \
-    --dicom-file scan.dcm \
-    --geojson-file annotations.json \
-    --output annotated.dcm
+aignostics wsi dicom geojson_import scan.dcm annotations.json
 ```
 
 ## Dependencies & Integration
@@ -285,8 +414,17 @@ aignostics wsi dicom-geojson-import \
 
 - `openslide-python` - Core WSI reading functionality
 - `Pillow` - Image processing and thumbnail generation
-- `pydicom` - DICOM file handling
+- `pydicom` - DICOM file handling (required for basic DICOM WSI operations)
 - `numpy` - Array manipulation for pixel data
+- `highdicom` - DICOM annotation/measurement features (optional, not available on Python 3.14+)
+
+**Python 3.14+ Compatibility:**
+
+The core WSI functionality (thumbnail generation, metadata extraction, DICOM file selection) works on Python 3.14+ without highdicom. Only the following CLI commands require highdicom and are unavailable on Python 3.14+:
+- `aignostics wsi dicom geojson_import` - Import GeoJSON to DICOM annotations
+- Detailed annotation/measurement inspection features
+
+The DICOM file selection logic (`select_dicom_files()`) works on all Python versions since it only depends on `pydicom`.
 
 ### Format Support Matrix
 

src/aignostics/wsi/_cli.py

@@ -132,6 +132,7 @@ def dicom_inspect(
     ],
     verbose: Annotated[bool, typer.Option(help="Verbose output")] = False,
     summary: Annotated[bool, typer.Option(help="Show only summary information")] = False,
+    wsi_only: Annotated[bool, typer.Option(help="Filter to WSI files only")] = False,
 ) -> None:  # pylint: disable=W0613
     """Inspect DICOM files at any hierarchy level."""
     if not _check_highdicom_available():
@@ -142,7 +143,7 @@ def dicom_inspect(
 
     try:
         with PydicomHandler.from_file(str(path)) as handler:
-            metadata = handler.get_metadata(verbose)
+            metadata = handler.get_metadata(verbose, wsi_only)
 
             if metadata["type"] == "empty":
                 console.print("[bold red]No DICOM files found in the specified path.[/bold red]")