Merge branch 'main' into HEA-809/add_regex_for_men_women_boys_girls

Author: girumb

apps/baseline/models.py

@@ -2,6 +2,7 @@
 Models for managing HEA Baseline Surveys
 """
 
+import datetime
 import numbers
 
 from django.conf import settings
@@ -131,7 +132,30 @@ class ExtraMeta:
         identifier = ["code"]
 
 
-class LivelihoodZoneBaselineManager(common_models.IdentifierManager):
+class LivelihoodZoneBaselineQuerySet(models.QuerySet):
+    """
+    QuerySet for LivelihoodZoneBaseline that provides temporal filtering methods.
+    """
+
+    def filter_current(self, as_of_date=None):
+        """
+        Return a queryset filtered to the baselines that are valid as of the date specified.
+        """
+        if not as_of_date:
+            as_of_date = datetime.date.today()
+        return self.filter(
+            (models.Q(valid_from_date__lte=as_of_date) | models.Q(valid_from_date__isnull=True))
+            & (models.Q(valid_to_date__gte=as_of_date) | models.Q(valid_to_date__isnull=True))
+        )
+
+    def current_all(self, as_of_date=None):
+        """
+        Return all the baselines that are valid as of the date specified.
+        """
+        return self.filter_current(as_of_date).all()
+
+
+class LivelihoodZoneBaselineManager(common_models.IdentifierManager.from_queryset(LivelihoodZoneBaselineQuerySet)):
     def get_by_natural_key(self, code: str, reference_year_end_date: str):
         return self.get(livelihood_zone__code=code, reference_year_end_date=reference_year_end_date)
 

apps/baseline/serializers.py

@@ -438,7 +438,11 @@ def get_characteristic_group(self, obj):
         """
         if obj.product and obj.product.cpc.startswith("L02"):
             return "Livestock"
-        return obj.wealth_characteristic.characteristic_group
+        return (
+            obj.wealth_characteristic.characteristic_group.code
+            if obj.wealth_characteristic.characteristic_group
+            else None
+        )
 
     product_common_name = serializers.CharField(source="product.common_name", read_only=True, allow_null=True)
     unit_of_measure_description = serializers.CharField(
@@ -542,7 +546,11 @@ def get_characteristic_group(self, obj):
         """
         if obj.product and obj.product.cpc.startswith("L02"):
             return "Livestock"
-        return obj.wealth_characteristic.characteristic_group
+        return (
+            obj.wealth_characteristic.characteristic_group.code
+            if obj.wealth_characteristic.characteristic_group
+            else None
+        )
 
     product_common_name = serializers.CharField(source="product.common_name", read_only=True, allow_null=True)
     unit_of_measure_description = serializers.CharField(

apps/baseline/tests/factories.py

@@ -119,8 +119,9 @@ class Meta:
     bss_language = factory.Iterator(["en", "pt", "es", "ar", "fr"])
     reference_year_start_date = factory.LazyAttribute(lambda o: o.reference_year_end_date - relativedelta(years=1))
     reference_year_end_date = factory.Sequence(lambda n: datetime.date(1900, 1, 1) + datetime.timedelta(days=n + 10))
-    valid_from_date = factory.Sequence(lambda n: datetime.date(1900, 1, 1) + datetime.timedelta(days=n))
-    valid_to_date = factory.Sequence(lambda n: datetime.date(1900, 1, 1) + datetime.timedelta(days=n + 10))
+    # Default to None so factory-created baselines are always valid regardless of as_of_date filter
+    valid_from_date = None
+    valid_to_date = None
     population_source = factory.Sequence(lambda n: f"population_source {n}")
     population_estimate = fuzzy.FuzzyInteger(500, 1000000)
     currency = factory.SubFactory(CurrencyFactory)

apps/baseline/tests/test_viewsets.py

@@ -2,13 +2,17 @@
 import json
 import logging
 import warnings
+from datetime import timedelta
 from io import StringIO
 
 import pandas as pd
 from bs4 import BeautifulSoup
 from django.contrib.auth.models import User
+from django.core.cache import cache
 from django.db.models import F
 from django.urls import reverse
+from django.utils.http import http_date
+from django.utils.timezone import now
 from rest_framework.test import APITestCase
 
 from baseline.models import (
@@ -19,6 +23,7 @@
 from common.tests.factories import ClassifiedProductFactory, CountryFactory
 from metadata.models import LivelihoodActivityScenario
 from metadata.tests.factories import (
+    CharacteristicGroupFactory,
     LivelihoodCategoryFactory,
     WealthCharacteristicFactory,
     WealthGroupCategoryFactory,
@@ -595,6 +600,125 @@ def test_filter_by_wealth_characteristic(self):
         self.assertEqual(response.status_code, 200)
         self.assertEqual(len(response.json()), 1)
 
+    def test_as_of_date_filter_returns_valid_baselines(self):
+        """
+        Test that the as_of_date filter returns only baselines valid as of the specified date.
+        """
+        today = datetime.date.today()
+
+        # Baseline that expired in the past
+        expired_baseline = LivelihoodZoneBaselineFactory(
+            valid_from_date=today - datetime.timedelta(days=365),
+            valid_to_date=today - datetime.timedelta(days=30),
+        )
+
+        # Baseline currently valid
+        current_baseline = LivelihoodZoneBaselineFactory(
+            valid_from_date=today - datetime.timedelta(days=30),
+            valid_to_date=today + datetime.timedelta(days=365),
+        )
+
+        # Baseline that starts in the future (not yet valid)
+        future_baseline = LivelihoodZoneBaselineFactory(
+            valid_from_date=today + datetime.timedelta(days=30),
+            valid_to_date=today + datetime.timedelta(days=365),
+        )
+
+        # Test default behavior (no as_of_date param) - should return all baselines
+        response = self.client.get(self.url)
+        self.assertEqual(response.status_code, 200)
+        baseline_ids = [b["id"] for b in response.json()]
+        self.assertIn(current_baseline.id, baseline_ids)
+        self.assertIn(expired_baseline.id, baseline_ids)
+        self.assertIn(future_baseline.id, baseline_ids)
+
+        # Test with explicit as_of_date=today - should filter to current baselines
+        response = self.client.get(self.url, {"as_of_date": today.isoformat()})
+        self.assertEqual(response.status_code, 200)
+        baseline_ids = [b["id"] for b in response.json()]
+        self.assertIn(current_baseline.id, baseline_ids)
+        self.assertNotIn(expired_baseline.id, baseline_ids)
+        self.assertNotIn(future_baseline.id, baseline_ids)
+
+        # Test with a past date when expired_baseline was still valid
+        # current_baseline hadn't started yet, so it should be excluded
+        past_date = today - datetime.timedelta(days=180)
+        response = self.client.get(self.url, {"as_of_date": past_date.isoformat()})
+        self.assertEqual(response.status_code, 200)
+        baseline_ids = [b["id"] for b in response.json()]
+        self.assertIn(expired_baseline.id, baseline_ids)
+        self.assertNotIn(current_baseline.id, baseline_ids)  # hadn't started yet
+        self.assertNotIn(future_baseline.id, baseline_ids)
+
+        # Test with a future date when future_baseline will be valid
+        future_date = today + datetime.timedelta(days=60)
+        response = self.client.get(self.url, {"as_of_date": future_date.isoformat()})
+        self.assertEqual(response.status_code, 200)
+        baseline_ids = [b["id"] for b in response.json()]
+        self.assertNotIn(expired_baseline.id, baseline_ids)
+        self.assertIn(current_baseline.id, baseline_ids)
+        self.assertIn(future_baseline.id, baseline_ids)
+
+        # Baseline with null valid_from_date (valid from the beginning of time)
+        baseline_no_from = LivelihoodZoneBaselineFactory(
+            valid_from_date=None,
+            valid_to_date=today + datetime.timedelta(days=365),
+        )
+        # Baseline with null valid_to_date (valid indefinitely)
+        baseline_no_to = LivelihoodZoneBaselineFactory(
+            valid_from_date=today - datetime.timedelta(days=365),
+            valid_to_date=None,
+        )
+        # Baseline with both dates null (always valid)
+        baseline_no_dates = LivelihoodZoneBaselineFactory(
+            valid_from_date=None,
+            valid_to_date=None,
+        )
+        # Test default behavior - all three should be returned
+        response = self.client.get(self.url)
+        self.assertEqual(response.status_code, 200)
+        baseline_ids = [b["id"] for b in response.json()]
+        self.assertIn(baseline_no_from.id, baseline_ids)
+        self.assertIn(baseline_no_to.id, baseline_ids)
+        self.assertIn(baseline_no_dates.id, baseline_ids)
+
+    def test_conditional_request_headers(self):
+        cache.clear()  # Clear cache to ensure clean state
+
+        # Test that 200 response includes ETag, Last-Modified, Cache-Control, and Expires headers
+        response = self.client.get(self.url)
+        self.assertEqual(response.status_code, 200)
+        self.assertIn("ETag", response.headers)
+        self.assertIn("Last-Modified", response.headers)
+        self.assertIn("Cache-Control", response.headers)
+        self.assertIn("Expires", response.headers)
+        self.assertTrue(response.headers["ETag"].startswith('W/"'))  # Weak ETag format
+
+        # Test If-None-Match returns 304 when not modified
+        etag = response.headers["ETag"]
+        cache.clear()
+        response = self.client.get(self.url, HTTP_IF_NONE_MATCH=etag)
+        self.assertEqual(response.status_code, 304)
+        self.assertIn("Cache-Control", response.headers)
+        self.assertIn("Expires", response.headers)
+
+        # Test If-None-Match returns 200 when data is modified
+        cache.clear()  # Clear cache before testing modified data
+        baseline = self.data[0]
+        baseline.population_source = "Updated source"
+        baseline.save()
+        response = self.client.get(self.url, HTTP_IF_NONE_MATCH=etag)
+        self.assertEqual(response.status_code, 200)
+        self.assertNotEqual(response.headers["ETag"], etag)
+
+        # Test If-Modified-Since with future date returns 304
+        cache.clear()
+        future_date = http_date((now() + timedelta(days=1)).timestamp())
+        response = self.client.get(self.url, HTTP_IF_MODIFIED_SINCE=future_date)
+        self.assertEqual(response.status_code, 304)
+        self.assertIn("Cache-Control", response.headers)
+        self.assertIn("Expires", response.headers)
+
 
 class LivelihoodZoneBaselineFacetedSearchViewTestCase(APITestCase):
     def setUp(self):
@@ -641,7 +765,8 @@ def test_search_with_product(self):
         # Apply the filters to the baseline
         baseline_url = reverse("livelihoodzonebaseline-list")
         response = self.client.get(
-            baseline_url, {search_data["products"][0]["filter"]: search_data["products"][0]["value"]}
+            baseline_url,
+            {search_data["products"][0]["filter"]: search_data["products"][0]["value"]},
         )
         self.assertEqual(response.status_code, 200)
         self.assertEqual(len(json.loads(response.content)), 2)
@@ -650,7 +775,10 @@ def test_search_with_product(self):
         self.assertTrue(any(d["name"] == self.baseline3.name for d in data))
         self.assertFalse(any(d["name"] == self.baseline2.name for d in data))
 
-        response = self.client.get(baseline_url, {search_data["items"][0]["filter"]: search_data["items"][0]["value"]})
+        response = self.client.get(
+            baseline_url,
+            {search_data["items"][0]["filter"]: search_data["items"][0]["value"]},
+        )
         self.assertEqual(response.status_code, 200)
         self.assertEqual(len(json.loads(response.content)), 1)
         data = json.loads(response.content)
@@ -1499,11 +1627,13 @@ def setUpTestData(cls):
             wealth_group_category=cls.very_poor_wg,
             community=cls.community,
         )
+        # Create characteristic group
+        cls.population_group = CharacteristicGroupFactory(code="Population", name_en="Population")
         # Create wealth characteristics
         cls.char_with_group = WealthCharacteristicFactory(
             code="household size",
             name_en="Household size",
-            characteristic_group="Population",
+            characteristic_group=cls.population_group,
         )
         cls.char_without_group = WealthCharacteristicFactory(
             code="other",

apps/baseline/viewsets.py

@@ -4,7 +4,10 @@
 from django.db.models import Expression, F, Q, Subquery, TextField, Value
 from django.db.models.functions import Coalesce, NullIf
 from django.utils import translation
+from django.utils.decorators import method_decorator
 from django.utils.translation import override
+from django.views.decorators.cache import cache_page
+from django.views.decorators.http import condition
 from django_filters import rest_framework as filters
 from django_filters.filters import CharFilter
 from rest_framework.permissions import AllowAny
@@ -13,7 +16,8 @@
 from rest_framework.views import APIView
 
 from common.fields import translation_fields
-from common.filters import MultiFieldFilter, UpperCaseFilter
+from common.filters import DefaultingDateFilter, MultiFieldFilter, UpperCaseFilter
+from common.utils import make_condition_funcs
 from common.viewsets import AggregatingViewSet, BaseModelViewSet
 from metadata.models import WealthGroupCategory
 
@@ -100,6 +104,9 @@
     WildFoodGatheringSerializer,
 )
 
+# Create condition functions for LivelihoodZoneBaseline endpoint caching
+get_baseline_etag, get_baseline_last_modified = make_condition_funcs(LivelihoodZoneBaseline)
+
 
 class SourceOrganizationFilterSet(filters.FilterSet):
     class Meta:
@@ -205,6 +212,10 @@ class Meta:
     wealth_characteristic = CharFilter(
         method="filter_by_wealth_characteristic", label="Filter by Wealth Characteristic"
     )
+    as_of_date = DefaultingDateFilter(
+        label="As of Date",
+        help_text="Filter baselines valid as of this date (YYYY-MM-DD format or special values like 'today').",
+    )
 
     def filter_by_product(self, queryset, name, value):
         """
@@ -271,6 +282,11 @@ def get_serializer_class(self):
             return LivelihoodZoneBaselineGeoSerializer  # Use GeoFeatureModelSerializer for GeoJSON
         return LivelihoodZoneBaselineSerializer
 
+    @method_decorator(cache_page(60 * 60 * 24))  # Cache on server for 24 hours - must be above condition per RFC 9110
+    @method_decorator(condition(etag_func=get_baseline_etag, last_modified_func=get_baseline_last_modified))
+    def list(self, request, *args, **kwargs):
+        return super().list(request, *args, **kwargs)
+
 
 class LivelihoodProductCategoryFilterSet(filters.FilterSet):
     class Meta:

apps/common/filters.py

@@ -8,14 +8,16 @@
 from django.core.validators import EMPTY_VALUES
 from django.db.models import F, Func, Q
 from django.forms import TextInput
-from django.forms.fields import ChoiceField, Field, MultipleChoiceField
+from django.forms.fields import ChoiceField, DateField, Field, MultipleChoiceField
 from django.forms.models import ModelMultipleChoiceField
 from django.utils.datastructures import MultiValueDict
 from django.utils.encoding import force_str
 from django_filters import ModelMultipleChoiceFilter, MultipleChoiceFilter
-from django_filters.filters import BooleanFilter, CharFilter, ChoiceFilter
+from django_filters.filters import BooleanFilter, CharFilter, ChoiceFilter, DateFilter
 from rest_framework.filters import OrderingFilter
 
+from .utils import DEFAULT_DATES
+
 logger = logging.getLogger(__name__)
 
 
@@ -238,3 +240,33 @@ def filter(self, qs, value):
         method = qs.exclude if exclude else qs.filter
 
         return method(**{self.field_name: ""})
+
+
+class DefaultingDateField(DateField):
+    """
+    A date field that accepts defaults like "last_month", "today"
+    """
+
+    def to_python(self, value):
+        if value in DEFAULT_DATES:
+            value = DEFAULT_DATES[value]()
+        return super().to_python(value)
+
+
+class DefaultingDateFilter(DateFilter):
+    """
+    A date filter that accepts defaults like "today"
+    """
+
+    field_class = DefaultingDateField
+
+    def filter(self, qs, value):
+        if value:
+            if self.lookup_expr in ["lte", "gte"]:
+                # period_date filter for start and end date with either gte or lte expression will fall here.
+                query = Q()
+                query = Q(**{self.field_name + "__" + self.lookup_expr: value})
+                return qs.filter(query)
+            else:
+                return qs.current_all(value)
+        return qs