eli-579 amending token parser to parse out function details

Author: eddalmond1

src/eligibility_signposting_api/services/processors/token_parser.py

@@ -13,20 +13,30 @@ class ParsedToken:
         Example: "PERSON" or "TARGET"
     attribute_name : str
         Example: "POSTCODE" or "RSV"
-    attribute_value : int
+    attribute_value : str | None
         Example: "LAST_SUCCESSFUL_DATE" if attribute_level is TARGET
-    format : str
+    format : str | None
         Example: "%d %B %Y" if DATE formatting is used
+    function_name : str | None
+        Example: "ADD_DAYS" for derived value functions
+    function_args : str | None
+        Example: "91" for ADD_DAYS(91)
     """
 
     attribute_level: str
     attribute_name: str
     attribute_value: str | None
     format: str | None
+    function_name: str | None = None
+    function_args: str | None = None
 
 
 class TokenParser:
     MIN_TOKEN_PARTS = 2
+    # Pattern for function calls like ADD_DAYS(91) - captures function name and args
+    FUNCTION_PATTERN = re.compile(r":([A-Z_]+)\(([^()]*)\)", re.IGNORECASE)
+    # Pattern for DATE format - special case as it's already supported
+    DATE_PATTERN = re.compile(r":DATE\(([^()]*)\)", re.IGNORECASE)
 
     @staticmethod
     def parse(token: str) -> ParsedToken:
@@ -35,8 +45,15 @@ def parse(token: str) -> ParsedToken:
         Strip the surrounding [[ ]]
         Check for empty body after stripping, e.g., '[[]]'
         Check for empty parts created by leading/trailing dots or tokens with no dot
-        Check if the name contains a date format
+        Check if the name contains a date format or function call
         Return a ParsedToken object
+
+        Supported formats:
+        - [[PERSON.AGE]] - Simple person attribute
+        - [[TARGET.COVID.LAST_SUCCESSFUL_DATE]] - Target attribute
+        - [[PERSON.DATE_OF_BIRTH:DATE(%d %B %Y)]] - With date formatting
+        - [[TARGET.COVID.NEXT_DOSE_DUE:ADD_DAYS(91)]] - Derived value function
+        - [[TARGET.COVID.NEXT_DOSE_DUE:ADD_DAYS(91):DATE(%d %B %Y)]] - Function with date format
         """
 
         token_body = token[2:-2]
@@ -53,14 +70,20 @@ def parse(token: str) -> ParsedToken:
         token_level = token_parts[0].upper()
         token_name = token_parts[-1]
 
-        format_match = re.search(r":DATE\(([^()]*)\)", token_name, re.IGNORECASE)
-        if not format_match and len(token_name.split(":")) > 1:
-            message = "Invalid token format."
-            raise ValueError(message)
+        # Extract function call (e.g., ADD_DAYS(91))
+        function_name, function_args = TokenParser._extract_function(token_name)
 
+        # Extract date format
+        format_match = TokenParser.DATE_PATTERN.search(token_name)
         format_str = format_match.group(1) if format_match else None
 
-        last_part = re.sub(r":DATE\([^)]*\)", "", token_name, flags=re.IGNORECASE)
+        # Validate format - if there's a colon but no valid pattern, it's invalid
+        if not format_match and not function_name and len(token_name.split(":")) > 1:
+            message = "Invalid token format."
+            raise ValueError(message)
+
+        # Remove function and date patterns to get the clean attribute name
+        last_part = TokenParser._clean_attribute_name(token_name)
 
         if len(token_parts) == TokenParser.MIN_TOKEN_PARTS:
             name = last_part.upper()
@@ -69,4 +92,42 @@ def parse(token: str) -> ParsedToken:
             name = token_parts[1].upper()
             value = last_part.upper()
 
-        return ParsedToken(attribute_level=token_level, attribute_name=name, attribute_value=value, format=format_str)
+        return ParsedToken(
+            attribute_level=token_level,
+            attribute_name=name,
+            attribute_value=value,
+            format=format_str,
+            function_name=function_name,
+            function_args=function_args,
+        )
+
+    @staticmethod
+    def _extract_function(token_name: str) -> tuple[str | None, str | None]:
+        """Extract function name and arguments from token name.
+
+        Args:
+            token_name: The last part of the token (e.g., 'NEXT_DOSE_DUE:ADD_DAYS(91)')
+
+        Returns:
+            Tuple of (function_name, function_args) or (None, None) if no function
+        """
+        # Find all function matches (excluding DATE which is handled separately)
+        for match in TokenParser.FUNCTION_PATTERN.finditer(token_name):
+            func_name = match.group(1).upper()
+            if func_name != "DATE":
+                return func_name, match.group(2)
+        return None, None
+
+    @staticmethod
+    def _clean_attribute_name(token_name: str) -> str:
+        """Remove function calls and date formatting from token name.
+
+        Args:
+            token_name: The raw token name with potential modifiers
+
+        Returns:
+            Clean attribute name
+        """
+        # Remove date format and other function calls
+        without_date = TokenParser.DATE_PATTERN.sub("", token_name)
+        return TokenParser.FUNCTION_PATTERN.sub("", without_date)

tests/unit/services/processors/test_token_parser.py

@@ -47,11 +47,17 @@ def test_parse_invalid_tokens_raises_error(self, token):
             "[[PERSON.DATE_OF_BIRTH:DATE(]]",
             "[[PERSON.DATE_OF_BIRTH:DATE)]]",
             "[[PERSON.DATE_OF_BIRTH:DATE]]",
-            "[[PERSON.DATE_OF_BIRTH:INVALID_FORMAT(abc)]]",
             "[[PERSON.DATE_OF_BIRTH:INVALID_FORMAT(a (b) c)]]",
             "[[PERSON.DATE_OF_BIRTH:DATE(a (b) c)]]",
         ],
     )
     def test_parse_invalid_token_format_raises_error(self, token):
         with pytest.raises(ValueError, match="Invalid token format."):
             TokenParser.parse(token)
+
+    def test_parse_function_token_valid(self):
+        """Test that valid function tokens are parsed correctly."""
+        # This used to be invalid, but now we support custom functions
+        parsed = TokenParser.parse("[[PERSON.DATE_OF_BIRTH:SOME_FUNC(abc)]]")
+        assert parsed.function_name == "SOME_FUNC"
+        assert parsed.function_args == "abc"

tests/unit/services/processors/test_token_parser_functions.py

@@ -0,0 +1,96 @@
+"""Tests for TokenParser with derived value function support."""
+
+from dataclasses import dataclass
+
+import pytest
+
+from eligibility_signposting_api.services.processors.token_parser import TokenParser
+
+
+@dataclass
+class ExpectedTokenResult:
+    """Expected result for a parsed token."""
+
+    level: str
+    name: str
+    value: str | None
+    function: str | None
+    args: str | None
+    date_format: str | None
+
+
+class TestTokenParserWithFunctions:
+    """Tests for parsing tokens with function calls like ADD_DAYS."""
+
+    @pytest.mark.parametrize(
+        ("token", "expected"),
+        [
+            # Basic ADD_DAYS function
+            (
+                "[[TARGET.COVID.NEXT_DOSE_DUE:ADD_DAYS(91)]]",
+                ExpectedTokenResult("TARGET", "COVID", "NEXT_DOSE_DUE", "ADD_DAYS", "91", None),
+            ),
+            # ADD_DAYS with date format
+            (
+                "[[TARGET.COVID.NEXT_DOSE_DUE:ADD_DAYS(91):DATE(%d %B %Y)]]",
+                ExpectedTokenResult("TARGET", "COVID", "NEXT_DOSE_DUE", "ADD_DAYS", "91", "%d %B %Y"),
+            ),
+            # Different vaccine type
+            (
+                "[[TARGET.RSV.NEXT_DOSE_DUE:ADD_DAYS(365)]]",
+                ExpectedTokenResult("TARGET", "RSV", "NEXT_DOSE_DUE", "ADD_DAYS", "365", None),
+            ),
+            # Case insensitive function name
+            (
+                "[[TARGET.COVID.NEXT_DOSE_DUE:add_days(91)]]",
+                ExpectedTokenResult("TARGET", "COVID", "NEXT_DOSE_DUE", "ADD_DAYS", "91", None),
+            ),
+            # Empty args (use default)
+            (
+                "[[TARGET.COVID.NEXT_DOSE_DUE:ADD_DAYS()]]",
+                ExpectedTokenResult("TARGET", "COVID", "NEXT_DOSE_DUE", "ADD_DAYS", "", None),
+            ),
+            # Person level with function (hypothetical future use)
+            (
+                "[[PERSON.SOME_DATE:ADD_DAYS(30)]]",
+                ExpectedTokenResult("PERSON", "SOME_DATE", None, "ADD_DAYS", "30", None),
+            ),
+        ],
+    )
+    def test_parse_tokens_with_functions(self, token: str, expected: ExpectedTokenResult):
+        """Test parsing tokens with function calls."""
+        parsed_token = TokenParser.parse(token)
+
+        assert parsed_token.attribute_level == expected.level
+        assert parsed_token.attribute_name == expected.name
+        assert parsed_token.attribute_value == expected.value
+        assert parsed_token.function_name == expected.function
+        assert parsed_token.function_args == expected.args
+        assert parsed_token.format == expected.date_format
+
+    def test_parse_without_function_has_none_function_fields(self):
+        """Test that tokens without functions have None for function fields."""
+        parsed = TokenParser.parse("[[TARGET.COVID.LAST_SUCCESSFUL_DATE]]")
+
+        assert parsed.function_name is None
+        assert parsed.function_args is None
+
+    def test_parse_date_format_not_treated_as_function(self):
+        """Test that DATE format is not treated as a derived function."""
+        parsed = TokenParser.parse("[[PERSON.DATE_OF_BIRTH:DATE(%d %B %Y)]]")
+
+        assert parsed.function_name is None
+        assert parsed.format == "%d %B %Y"
+
+    @pytest.mark.parametrize(
+        "token",
+        [
+            "[[TARGET.COVID.NEXT_DOSE_DUE:ADD_DAYS]]",  # Missing parentheses
+            "[[TARGET.COVID.NEXT_DOSE_DUE:ADD_DAYS(]]",  # Unclosed parenthesis
+            "[[TARGET.COVID.NEXT_DOSE_DUE:ADD_DAYS)]]",  # No opening parenthesis
+        ],
+    )
+    def test_parse_invalid_function_format_raises_error(self, token):
+        """Test that malformed function calls raise errors."""
+        with pytest.raises(ValueError, match="Invalid token format"):
+            TokenParser.parse(token)