Merge pull request #699 from tiran/annotations

feat: Add annotation support for packages and variants

Author: mergify[bot]

src/fromager/packagesettings.py

@@ -102,6 +102,54 @@ def _validate_envkey(v: typing.Any) -> str:
 GlobalChangelog = Mapping[Variant, list[str]]
 VariantChangelog = Mapping[PackageVersion, list[str]]
 
+# Annotations
+RawAnnotations = Mapping[str, str]
+
+
+class Annotations(Mapping):
+    """Read-only mapping for package annotations"""
+
+    __slots__ = "_mapping"
+
+    def __init__(
+        self,
+        package: RawAnnotations | None = None,
+        variant: RawAnnotations | None = None,
+    ) -> None:
+        self._mapping: RawAnnotations = {}
+        if package:
+            self._mapping.update(package)
+        if variant:
+            self._mapping.update(variant)
+
+    def __getitem__(self, key: str) -> str:
+        return self._mapping[key]
+
+    def __iter__(self) -> typing.Iterator[str]:
+        return iter(self._mapping)
+
+    def __len__(self) -> int:
+        return len(self._mapping)
+
+    def __repr__(self):
+        return repr(self._mapping)
+
+    def getbool(self, key: str) -> bool:
+        """Get bool from string value
+
+        raises :exc:`KeyError` when key is missing and :exc`ValueError` when
+        the value is not 1, true, on, yes, 0, false, off, no.
+        """
+        value = self[key]
+        match value.lower():
+            case "1" | "true" | "on" | "yes":
+                return True
+            case "0" | "false" | "off" | "no":
+                return False
+            case _:
+                raise ValueError(value)
+
+
 # common settings
 MODEL_CONFIG = pydantic.ConfigDict(
     # don't accept unknown keys
@@ -279,6 +327,13 @@ class VariantInfo(pydantic.BaseModel):
 
     model_config = MODEL_CONFIG
 
+    annotations: RawAnnotations | None = None
+    """Arbitrary metadata for variants
+
+    Variant annotation keys have a higher precedence than package
+    annotation keys.
+    """
+
     env: EnvVars = Field(default_factory=dict)
     """Additional env vars (overrides package env vars)"""
 
@@ -362,6 +417,9 @@ class PackageSettings(pydantic.BaseModel):
     has_config: bool
     """package has override setting"""
 
+    annotations: RawAnnotations | None = None
+    """Arbitrary metadata for a package"""
+
     build_dir: BuildDirectory | None = None
     """sub-directory with setup.py or pyproject.toml"""
 
@@ -568,6 +626,7 @@ def __init__(self, settings: Settings, ps: PackageSettings) -> None:
         self._ps = ps
         self._plugin_module: types.ModuleType | None | typing.Literal[False] = False
         self._patches: PatchMap | None = None
+        self._annotations: Annotations | None = None
 
     @property
     def package(self) -> NormalizedName:
@@ -579,6 +638,31 @@ def variant(self) -> Variant:
         """Variant name"""
         return self._variant
 
+    @property
+    def annotations(self) -> Annotations:
+        """Get Package and variant annotations
+
+        Annotations can be used to attach arbitrary metadata to packages and
+        package variants. The feature is inspired by Kubernetes's
+        annotations. Variant keys have a higher precedence than package keys.
+
+        The prefix ``fromager.`` is reserved for future use by Fromager.
+
+        ::
+
+           annotations:
+             "downstream.maintainer": "Platform Team"
+           variants:
+             cuda:
+               annotations:
+                 "downstream.maintainer": "CUDA Accelerator Team"
+        """
+        if self._annotations is None:
+            vi = self._ps.variants.get(self.variant)
+            va = vi.annotations if vi is not None else None
+            self._annotations = Annotations(self._ps.annotations, va)
+        return self._annotations
+
     @property
     def plugin(self) -> types.ModuleType | None:
         """Get Fromager plugin module"""

tests/test_packagesettings.py

@@ -10,6 +10,7 @@
 
 from fromager import build_environment, context
 from fromager.packagesettings import (
+    Annotations,
     BuildDirectory,
     EnvVars,
     GitOptions,
@@ -28,6 +29,10 @@
 TEST_PREBUILT_PKG = "test-prebuilt-pkg"
 
 FULL_EXPECTED: dict[str, typing.Any] = {
+    "annotations": {
+        "fromager.test.value": "somevalue",
+        "fromager.test.override": "variant override",
+    },
     "build_dir": pathlib.Path("python"),
     "build_options": {
         "build_ext_parallel": True,
@@ -76,16 +81,23 @@
     },
     "variants": {
         "cpu": {
+            "annotations": {
+                "fromager.test.override": "cpu override",
+            },
             "env": {"EGG": "spam ${EGG}", "EGG_AGAIN": "$EGG"},
             "wheel_server_url": "https://wheel.test/simple",
             "pre_built": False,
         },
         "rocm": {
+            "annotations": {
+                "fromager.test.override": "amd override",
+            },
             "env": {"SPAM": ""},
             "wheel_server_url": None,
             "pre_built": True,
         },
         "cuda": {
+            "annotations": None,
             "env": {},
             "wheel_server_url": None,
             "pre_built": False,
@@ -95,6 +107,7 @@
 
 EMPTY_EXPECTED: dict[str, typing.Any] = {
     "name": "test-empty-pkg",
+    "annotations": None,
     "build_dir": None,
     "build_options": {
         "build_ext_parallel": False,
@@ -130,6 +143,7 @@
 
 PREBUILT_PKG_EXPECTED: dict[str, typing.Any] = {
     "name": "test-prebuilt-pkg",
+    "annotations": None,
     "build_dir": None,
     "build_options": {
         "build_ext_parallel": False,
@@ -164,6 +178,7 @@
     },
     "variants": {
         "cpu": {
+            "annotations": None,
             "env": {},
             "pre_built": True,
             "wheel_server_url": None,
@@ -722,3 +737,50 @@ def test_package_build_info_exclusive_build(testdata_context: context.WorkContex
 def test_resolver_dist_validator():
     with pytest.raises(pydantic.ValidationError):
         ResolverDist(include_wheels=False, ignore_platform=True)
+
+
+def test_annotation_type() -> None:
+    ann = Annotations(None, None)
+    assert not ann
+    assert len(ann) == 0
+    assert ann == {}
+    with pytest.raises(TypeError):
+        ann["key"] = "value"  # type: ignore
+
+    ann = Annotations({"ka": "va", "kb": "vb"}, {"kb": "otherb", "kc": "vc"})
+    assert ann
+    assert len(ann) == 3
+    assert ann == {"ka": "va", "kb": "otherb", "kc": "vc"}
+
+    ann = Annotations({"t": "yes", "f": "no", "invalid": "invalid"}, {})
+    assert ann.getbool("t") is True
+    assert ann.getbool("f") is False
+    with pytest.raises(ValueError):
+        ann.getbool("invalid")
+    with pytest.raises(KeyError):
+        ann.getbool("missing")
+
+
+def test_pbi_annotations(testdata_context: context.WorkContext) -> None:
+    pbi = testdata_context.settings.package_build_info(TEST_PKG)
+    assert pbi.annotations == {
+        "fromager.test.value": "somevalue",
+        "fromager.test.override": "cpu override",
+    }
+
+    testdata_context.settings.variant = Variant("cuda")
+    pbi = testdata_context.settings.package_build_info(TEST_PKG)
+    assert pbi.annotations == {
+        "fromager.test.value": "somevalue",
+        "fromager.test.override": "variant override",
+    }
+
+    testdata_context.settings.variant = Variant("rocm")
+    pbi = testdata_context.settings.package_build_info(TEST_PKG)
+    assert pbi.annotations == {
+        "fromager.test.value": "somevalue",
+        "fromager.test.override": "amd override",
+    }
+
+    pbi = testdata_context.settings.package_build_info(TEST_EMPTY_PKG)
+    assert pbi.annotations == {}

tests/testdata/context/overrides/settings/test_pkg.yaml

@@ -1,3 +1,6 @@
+annotations:
+    fromager.test.value: somevalue
+    fromager.test.override: variant override
 build_dir: python
 build_options:
     build_ext_parallel: true
@@ -38,11 +41,15 @@ resolver_dist:
     ignore_platform: true
 variants:
     cpu:
+        annotations:
+            fromager.test.override: cpu override
         env:
             EGG: "spam ${EGG}"
             EGG_AGAIN: "$EGG"
         wheel_server_url: https://wheel.test/simple
     rocm:
+        annotations:
+            fromager.test.override: amd override
         env:
             SPAM: ""
         pre_built: True