Merge branch 'main' into HEA-984/Livelihood_activity_valid_instances_errors_for_income_not_rounding_error

girumb · girumb · commit 831a737c2a0e · 2026-03-27T12:33:57.000+03:00
diff --git a/apps/baseline/admin.py b/apps/baseline/admin.py
@@ -655,6 +655,7 @@ def get_fieldsets(self, request, obj=None):
                 "Payment",
                 {
                     "fields": [
+                        "payment_product",
                         "people_per_household",
                         "times_per_month",
                         "months_per_year",
diff --git a/apps/baseline/migrations/0028_paymentinkind_payment_product.py b/apps/baseline/migrations/0028_paymentinkind_payment_product.py
@@ -0,0 +1,29 @@
+# Generated by Django 5.2.11 on 2026-03-19 17:20
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("baseline", "0027_alter_seasonalactivity_livelihood_zone_baseline"),
+        ("common", "0011_additionalpermissions"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="paymentinkind",
+            name="payment_product",
+            field=models.ForeignKey(
+                blank=True,
+                db_column="payment_product_code",
+                help_text="Product provided as payment, typically a staple food item in exchange for labor",
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="payments_in_kind",
+                to="common.classifiedproduct",
+                verbose_name="Payment Product",
+            ),
+        ),
+    ]
diff --git a/apps/baseline/models.py b/apps/baseline/models.py
@@ -1830,9 +1830,30 @@ class PaymentInKind(LivelihoodActivity):
     Food items that contribute to nutrition by households in a Wealth Group received in exchange for labor.
 
     Stored on the BSS 'Data' worksheet in the 'Payment In Kind' section, typically starting around Row 514.
+
+    Payments in kind contain two Product references: the `product` is the actual labor or service performed,
+    and the `payment_product` is the food item received as payment. The `quantity_produced` is the amount of the
+    `payment_product` received  annually, and the `kcals_consumed` should be the `quantity_produced` multiplied
+    by the `payment_product.kcals_per_unit`.
     """
 
-    # Production calculation/validation is `people_per_household * times_per_month * months_per_year`
+    # Note that payment_product is unusual because it is normally set using a label in Column A in the BSS,
+    # and so will have the same value for every Livelihood Activity. It could be considered part of the
+    # Livelihood Strategy rather than the individual Livelihood Activity, but it is the only example of an
+    # attribute that is logically part of the Livelihood Strategy but only for applies to a specific
+    # strategy type. We have included it in the PaymentInKind LivelihoodActivity for now, because many
+    # Livelihood Activity subclasses have additional attributes.
+    payment_product = models.ForeignKey(
+        ClassifiedProduct,
+        db_column="payment_product_code",
+        blank=True,
+        null=True,
+        on_delete=models.PROTECT,
+        verbose_name=_("Payment Product"),
+        help_text=_("Product provided as payment, typically a staple food item in exchange for labor"),
+        related_name="payments_in_kind",
+    )
+    # Production calculation/validation is `payment_per_time * people_per_household * times_per_month * months_per_year`
     payment_per_time = models.FloatField(
         blank=True,
         null=True,  # Not required if people_per_household or times_per_month is null
diff --git a/apps/baseline/serializers.py b/apps/baseline/serializers.py
@@ -834,12 +834,24 @@ class PaymentInKindSerializer(LivelihoodActivitySerializer):
     class Meta:
         model = PaymentInKind
         fields = LivelihoodActivitySerializer.Meta.fields + [
+            "payment_product",
+            "payment_product_common_name",
+            "payment_product_description",
             "payment_per_time",
             "people_per_household",
             "times_per_month",
             "months_per_year",
         ]
 
+    payment_product_common_name = serializers.SerializerMethodField()
+    payment_product_description = serializers.SerializerMethodField()
+
+    def get_payment_product_common_name(self, obj):
+        return obj.payment_product.common_name if obj.payment_product else None
+
+    def get_payment_product_description(self, obj):
+        return obj.payment_product.description_en if obj.payment_product else None
+
 
 class ReliefGiftOtherSerializer(LivelihoodActivitySerializer):
     class Meta:
diff --git a/apps/baseline/tests/test_viewsets.py b/apps/baseline/tests/test_viewsets.py
@@ -3740,6 +3740,9 @@ def test_get_record(self):
             "expenditure",
             "kcals_consumed",
             "percentage_kcals",
+            "payment_product",
+            "payment_product_common_name",
+            "payment_product_description",
             "payment_per_time",
             "people_per_household",
             "times_per_month",
diff --git a/apps/metadata/migrations/0016_activitylabel_payment_product.py b/apps/metadata/migrations/0016_activitylabel_payment_product.py
@@ -0,0 +1,28 @@
+# Generated by Django 5.2.11 on 2026-03-20 02:19
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("common", "0011_additionalpermissions"),
+        ("metadata", "0015_season_purpose"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="activitylabel",
+            name="payment_product",
+            field=models.ForeignKey(
+                blank=True,
+                db_column="payment_product_code",
+                null=True,
+                on_delete=django.db.models.deletion.RESTRICT,
+                related_name="payment_activity_labels",
+                to="common.classifiedproduct",
+                verbose_name="Payment Product",
+            ),
+        ),
+    ]
diff --git a/apps/metadata/models.py b/apps/metadata/models.py
@@ -473,6 +473,15 @@ class LivelihoodActivityType(models.TextChoices):
         related_name="activity_labels",
         verbose_name=_("Product"),
     )
+    payment_product = models.ForeignKey(
+        ClassifiedProduct,
+        db_column="payment_product_code",
+        null=True,
+        blank=True,
+        on_delete=models.RESTRICT,
+        related_name="payment_activity_labels",
+        verbose_name=_("Payment Product"),
+    )
     unit_of_measure = models.ForeignKey(
         UnitOfMeasure,
         db_column="unit_code",
diff --git a/pipelines/assets/livelihood_activity.py b/pipelines/assets/livelihood_activity.py
@@ -244,7 +244,9 @@ def get_livelihood_activity_regexes() -> list:
     placeholder_patterns = {
         "label_pattern": r"[a-zà-ÿ][a-zà-ÿ1-9',/ \.\>\-\(\)]+?",
         "product_pattern": r"(?P<product_id>[a-zà-ÿ][a-zà-ÿ1-9',/ \.\>\-\(\)]+?)",
-        "season_pattern": r"(?P<season>season [12]|saison [12]|[12][a-z] season||[12][a-zà-ÿ] saison|r[eé]colte principale|principale r[eé]colte|gu|deyr+?)",  # NOQA: E501
+        "payment_product_pattern": r"(?P<payment_product_id>[a-zà-ÿ][a-zà-ÿ1-9',/ \.\>\-\(\)]+?)",
+        "labor_pattern": r"(?P<product_id>(?:labou?r|travail|main-d'œuvre|pre-harvest labou?r|labour:? pre-harvest|harvest labou?r|labour:? harvest|post-harvest labou?r|labour:? post-harvest|travail:? pre-r[eéè]colte) *[:-]? *[a-zà-ÿ][a-zà-ÿ1-9',/ \.\>\-\(\)]+?)",
+        "season_pattern": r"(?P<season>season [12]|saison [12]|[12][a-z] season||[12][a-zà-ÿ] saison|r[eé]colte principale|principale r[eé]colte|gu|deyr+?)",
         "additional_identifier_pattern": r"\(?(?P<additional_identifier>rainfed|irrigated|pluviale?|irriguée|submersion libre|submersion contrôlée|flottant)\)?",
         "age_gender_pattern": age_gender_pattern,
         "unit_of_measure_pattern": r"(?P<unit_of_measure_id>[a-z]+)",
@@ -277,6 +279,7 @@ def get_livelihood_activity_regular_expression_attributes(label: str) -> dict:
         "strategy_type": None,
         "is_start": None,
         "product_id": None,
+        "payment_product_id": None,
         "unit_of_measure_id": None,
         "currency_id": None,
         "season": None,
@@ -341,6 +344,7 @@ def get_livelihood_activity_label_map(activity_type: str) -> dict[str, dict]:
             "strategy_type",
             "is_start",
             "product_id",
+            "payment_product_id",
             "unit_of_measure_id",
             "currency_id",
             "season",
@@ -416,6 +420,10 @@ def get_all_label_attributes(
     all_label_attributes = labels.apply(lambda x: get_label_attributes(x, activity_type)).fillna("")
     all_label_attributes = classifiedproductlookup.do_lookup(all_label_attributes, "product_id", "product_id")
     all_label_attributes["product_id"] = all_label_attributes["product_id"].replace(pd.NA, None)
+    all_label_attributes = classifiedproductlookup.do_lookup(
+        all_label_attributes, "payment_product_id", "payment_product_id"
+    )
+    all_label_attributes["payment_product_id"] = all_label_attributes["payment_product_id"].replace(pd.NA, None)
     all_label_attributes = unitofmeasurelookup.do_lookup(
         all_label_attributes, "unit_of_measure_id", "unit_of_measure_id"
     )
@@ -461,7 +469,9 @@ def get_all_label_attributes(
             columns = ["activity_label", "status", "strategy_type", "product_id", "season_original", "country_id"]
             if livelihood_zone_id:
                 columns.append("livelihood_zone_id")
-            raise ValueError("Unrecognized seasons in labels:\n" + unrecognized_seasons_df[columns].to_markdown())
+            raise ValueError(
+                f"Unrecognized seasons in {activity_type} labels:\n" + unrecognized_seasons_df[columns].to_markdown()
+            )
 
     # Make sure we keep the same index so we can match by row number
     all_label_attributes.index = labels.index
@@ -541,6 +551,9 @@ def livelihood_activity_label_recognition_dataframe(
             recognized_attributes_df["product_name"] = (
                 recognized_attributes_df["product_id"].map(product_name_map).fillna("")
             )
+            recognized_attributes_df["payment_product_name"] = (
+                recognized_attributes_df["payment_product_id"].map(product_name_map).fillna("")
+            )
 
             # Join the recognized attributes to the label dataframe
             label_df["activity_type"] = activity_type
@@ -554,10 +567,15 @@ def livelihood_activity_label_recognition_dataframe(
     regex_attributes_df = pd.DataFrame.from_records(
         all_labels_df["label"].astype(str).map(get_livelihood_activity_regular_expression_attributes)
     )
-    regex_attributes_df = ClassifiedProductLookup(require_match=False).do_lookup(
-        regex_attributes_df, "product_id", "product_id"
+    classifiedproductlookup = ClassifiedProductLookup(require_match=False)
+    regex_attributes_df = classifiedproductlookup.do_lookup(regex_attributes_df, "product_id", "product_id")
+    regex_attributes_df = classifiedproductlookup.do_lookup(
+        regex_attributes_df, "payment_product_id", "payment_product_id"
     )
     regex_attributes_df["product_name"] = regex_attributes_df["product_id"].map(product_name_map).fillna("")
+    regex_attributes_df["payment_product_name"] = (
+        regex_attributes_df["payment_product_id"].map(product_name_map).fillna("")
+    )
     all_labels_df = all_labels_df.join(
         regex_attributes_df,
         how="left",
@@ -574,6 +592,7 @@ def livelihood_activity_label_recognition_dataframe(
             "strategy_type",
             "is_start",
             "product_id",
+            "payment_product_id",
             "unit_of_measure_id",
             "currency_id",
             "season",
@@ -583,6 +602,7 @@ def livelihood_activity_label_recognition_dataframe(
         )
     )
     db_labels_df["product_name"] = db_labels_df["product_id"].map(product_name_map).fillna("")
+    db_labels_df["payment_product_name"] = db_labels_df["payment_product_id"].map(product_name_map).fillna("")
     all_labels_df = all_labels_df.join(
         db_labels_df.set_index(["label_lower", "activity_type"]),
         on=("label_lower", "activity_type"),
@@ -633,6 +653,9 @@ def livelihood_activity_label_recognition_dataframe(
             how="left",
         )
         activity_type_label_df["product_name"] = activity_type_label_df["product_id"].map(product_name_map).fillna("")
+        activity_type_label_df["payment_product_name"] = (
+            activity_type_label_df["payment_product_id"].map(product_name_map).fillna("")
+        )
         summary_label_dfs.append(activity_type_label_df)
 
     # Concatenate all the activity type dataframes back into a single dataframe and put
@@ -655,13 +678,15 @@ def livelihood_activity_label_recognition_dataframe(
             "strategy_type",
             "attribute",
             "product_name",
+            "payment_product_name",
             "unit_of_measure_id_original",
             "currency_id",
             "season",
             "additional_identifier",
             "notes",
             "household_labor_provider",
             "product_id",
+            "payment_product_id",
             "activity_label",
         ]
     ]
@@ -671,11 +696,14 @@ def livelihood_activity_label_recognition_dataframe(
         all_labels_df[
             [
                 "label_lower",
+                "activity_type",
                 "activity_label_regex",
                 "strategy_type_regex",
                 "is_start_regex",
                 "product_id_regex",
                 "product_name_regex",
+                "payment_product_id_regex",
+                "payment_product_name_regex",
                 "unit_of_measure_id_regex",
                 "season_regex",
                 "additional_identifier_regex",
@@ -685,6 +713,8 @@ def livelihood_activity_label_recognition_dataframe(
                 "is_start_db",
                 "product_id_db",
                 "product_name_db",
+                "payment_product_id_db",
+                "payment_product_name_db",
                 "unit_of_measure_id_db",
                 "season_db",
                 "additional_identifier_db",
@@ -693,8 +723,8 @@ def livelihood_activity_label_recognition_dataframe(
             ]
         ]
         .drop_duplicates()
-        .set_index("label_lower"),
-        on="label",
+        .set_index(["label_lower", "activity_type"]),
+        on=["label", "activity_type"],
         how="inner",
     )
 
@@ -1570,6 +1600,33 @@ def get_instances_from_dataframe(
                                     livelihood_strategy["attribute_rows"][attribute],
                                 )
                             )
+                    elif attribute in activity_field_names and value:
+                        # This is an attribute for the LivelihoodActivity rather than the LivelihoodStrategy,
+                        # so add it to the livelihood activities for this strategy.
+                        if attribute not in livelihood_strategy["attribute_rows"]:
+                            livelihood_strategy["attribute_rows"][attribute] = row
+                            for livelihood_activity in livelihood_activities_for_strategy:
+                                livelihood_activity[attribute] = value
+                        elif livelihood_activities_for_strategy[0][attribute] != value:
+                            errors.append(
+                                "Found different value '%s' from row %s for existing attribute '%s' with value '%s' from row %s for activity with label '%s'"
+                                % (
+                                    value,
+                                    row,
+                                    attribute,
+                                    livelihood_activities_for_strategy[0][attribute],
+                                    livelihood_strategy["attribute_rows"][attribute],
+                                    label,
+                                )
+                            )
+                    elif (
+                        attribute.endswith("_original")
+                        and attribute.removesuffix("_original") in activity_field_names
+                        and value
+                    ):
+                        # Keep the original value of the attribute to aid trouble-shooting.
+                        for livelihood_activity in livelihood_activities_for_strategy:
+                            livelihood_activity[attribute] = value
 
             # Update the LivelihoodActivity records
             if any(value for value in df.loc[row, "B":].astype(str).str.strip()):
diff --git a/pipelines/assets/livelihood_activity_regexes.json b/pipelines/assets/livelihood_activity_regexes.json
@@ -308,13 +308,13 @@
         "price"
     ],
     [
-        "paiement en {unit_of_measure_pattern} par fois \\({product_pattern}\\)",
+        "paiement en {unit_of_measure_pattern} par fois \\({payment_product_pattern}\\)",
         null,
         false,
         "payment_per_time"
     ],
     [
-        "payment in {unit_of_measure_pattern} per time \\({product_pattern}\\)",
+        "payment in {unit_of_measure_pattern} per time \\({payment_product_pattern}\\)",
         null,
         false,
         "payment_per_time"
@@ -505,12 +505,6 @@
         true,
         "quantity_produced"
     ],
-    [
-        "{product_pattern}{separator_pattern} {additional_identifier_pattern}: {unit_of_measure_pattern} produced",
-        null,
-        true,
-        "quantity_produced"
-    ],
     [
         "{product_pattern}{separator_pattern} {unit_of_measure_pattern} produced",
         null,
@@ -955,6 +949,12 @@
         true,
         "quantity_produced_or_purchased"
     ],
+    [
+        "{labor_pattern}",
+        null,
+        true,
+        "activity_notes"
+    ],
     [
         "autre{separator_pattern} {product_pattern}",
         null,
diff --git a/pipelines/jobs/metadata.py b/pipelines/jobs/metadata.py
diff --git a/pipelines_tests/test_assets/test_livelihood_activity_regexes.json b/pipelines_tests/test_assets/test_livelihood_activity_regexes.json

Original file line number	Diff line number	Diff line change
`@@ -655,6 +655,7 @@ def get_fieldsets(self, request, obj=None):`
`655`	`655`	`"Payment",`
`656`	`656`	`{`
`657`	`657`	`"fields": [`
	`658`	`+ "payment_product",`
`658`	`659`	`"people_per_household",`
`659`	`660`	`"times_per_month",`
`660`	`661`	`"months_per_year",`