[uss_qualifier/flight_data_resource] Validate flights, fix flight data, test flight validation

monitoring/uss_qualifier/resources/netrid/flight_data_resources.py

@@ -11,8 +11,10 @@
     TestFlightDetails,
 )
 
+from monitoring.monitorlib.rid import RIDVersion
 from monitoring.monitorlib.rid_automated_testing.injection_api import TestFlight
 from monitoring.uss_qualifier.resources.files import load_content, load_dict
+from monitoring.uss_qualifier.resources.netrid.evaluation import EvaluationConfiguration
 from monitoring.uss_qualifier.resources.netrid.flight_data import (
     FlightDataSpecification,
     FlightRecordCollection,
@@ -24,6 +26,7 @@
     get_flight_records,
 )
 from monitoring.uss_qualifier.resources.resource import Resource
+from monitoring.uss_qualifier.scenarios.scenario import TestScenario
 
 
 class FlightDataResource(Resource[FlightDataSpecification]):
@@ -190,34 +193,40 @@ def _validate_flights(self):
     def _validate_flight(self, flight):
         """Ensure flight data is valid"""
 
-        for state in flight.states:
-            # RIDAircraftState don't enforce values for thoses fields so we can
-            # create invalid flight on purpose, but we want then to be valid
-            # when coming from a source.
-            # See https://github.com/interuss/uas_standards/blob/main/src/uas_standards/interuss/automated_testing/rid/v1/injection.py#L412
-
-            for field in [
-                "timestamp",
-                "timestamp_accuracy",
-                "speed",
-                "vertical_speed",
-                "track",
-                "speed_accuracy",
-                "position",
-            ]:
-                if not state.has_field_with_value(field) or state[field] is None:
-                    raise Exception(
-                        f"Mandatory field {field} not found in state {state}"
-                    )
-
-            for field in ["accuracy_h", "accuracy_v"]:
-                if (
-                    not state.position.has_field_with_value(field)
-                    or state.position[field] is None
-                ):
-                    raise Exception(
-                        f"Mandatory field position.{field} not found in state {state}"
-                    )
+        from monitoring.uss_qualifier.scenarios.astm.netrid.common_dictionary_evaluator import (
+            RIDCommonDictionaryEvaluator,
+        )  # Circular import
+
+        # We pass the flight throught a "normal" TestFlight (some processing is
+        # done inside)
+        details = TestFlightDetails(
+            effective_after=StringBasedDateTime(arrow.utcnow()),
+            details=flight.flight_details,
+        )
+
+        test_flight = TestFlight(
+            injection_id=str(uuid.uuid4()),
+            telemetry=flight.states[::],
+            details_responses=[details],
+            aircraft_type=flight.aircraft_type,
+            filter_invalid_telemetry=False,
+        )
+
+        class DummyTestScenario(TestScenario):
+            def __init__(self):
+                pass
+
+            def run(self, *args, **kwargs):
+                pass
+
+        # We ask the evaluator to process it
+        evaluator = RIDCommonDictionaryEvaluator(
+            EvaluationConfiguration(), DummyTestScenario(), RIDVersion.f3411_22a
+        )
+        for state in test_flight.raw_telemetry or []:
+            evaluator.evaluate_injected_flight(
+                state, test_flight, test_flight.details_responses[0].details
+            )
 
 
 class FlightDataStorageSpecification(ImplicitDict):

monitoring/uss_qualifier/resources/netrid/flight_data_resources_test.py

@@ -25,10 +25,33 @@ def test_record():
 
 
 def test_invalid_record():
-    # accuracy_h is a field in position, speed_accuracy in state.
     for file in [
-        "test/invalid_no_accuracy_h.kml",
-        "test/invalid_no_speed_accuracy.kml",
+        "test/invalid_wrong_ua_type.json",
+        "test/invalid_no_timestamp.json",
+        "test/invalid_wrong_timestamp.json",
+        "test/invalid_no_timestamp_accuracy.json",
+        "test/invalid_wrong_timestamp_accuracy.json",
+        "test/invalid_wrong_operational_status.json",
+        "test/invalid_no_alt.json",
+        "test/invalid_no_accuracy_v.json",
+        "test/invalid_wrong_accuracy_v.json",
+        "test/invalid_no_accuracy_h.json",
+        "test/invalid_wrong_accuracy_h.json",
+        "test/invalid_no_speed_accuracy.json",
+        "test/invalid_wrong_speed_accuracy.json",
+        "test/invalid_no_vertical_speed.json",
+        "test/invalid_wrong_vertical_speed.json",
+        "test/invalid_no_speed.json",
+        "test/invalid_wrong_speed.json",
+        "test/invalid_no_track.json",
+        "test/invalid_wrong_track.json",
+        "test/invalid_no_height.json",
+        "test/invalid_no_height_type.json",
+        "test/invalid_wrong_height_type.json",
+        "test/invalid_no_uas_id.json",
+        "test/invalid_wrong_serial_number.json",
+        "test/invalid_wrong_registration_id.json",
+        "test/invalid_wrong_utm_id.json",
     ]:
         specs = FlightDataSpecification(
             record_source=ExternalFile(path=f"file://./test_data/{file}")
@@ -55,6 +78,8 @@ def test_invalid_kmls():
     for file in [
         "test/invalid_no_accuracy_h.kml",
         "test/invalid_no_speed_accuracy.kml",
+        "test/invalid_wrong_serial_number.kml",
+        "test/invalid_wrong_ua_type.kml",
     ]:
         specs = FlightDataSpecification(
             kml_source=FlightDataKMLFileConfiguration(

monitoring/uss_qualifier/resources/netrid/simulation/operator_flight_details.py

@@ -17,8 +17,10 @@ def generate_serial_number(self):
         return str(SerialNumber.generate_valid())
 
     def generate_registration_number(self, prefix="CHE"):
+        if not prefix.endswith("."):
+            prefix = f"{prefix}."
         registration_number = prefix + "".join(
-            self.random.choices(string.ascii_lowercase + string.digits, k=13)
+            self.random.choices(string.ascii_uppercase + string.digits, k=13)
         )
         return registration_number
 

monitoring/uss_qualifier/scenarios/astm/netrid/common_dictionary_evaluator.py

@@ -54,6 +54,10 @@
 T2 = TypeVar("T2")
 
 
+class NoObservedFlight(ValueError):
+    pass
+
+
 class RIDCommonDictionaryEvaluator:
     flight_evaluators = [
         "_evaluate_ua_type",
@@ -93,6 +97,45 @@ def __init__(
         self._test_scenario = test_scenario
         self._rid_version = rid_version
 
+    def evaluate_injected_flight(
+        self, injected_telemetry, injected_flight, injected_flight_details
+    ) -> None:
+        """Helper for generator of flights that raise an exception if an injected flight is invalid"""
+
+        for generics_evaluator in self.flight_evaluators:
+            try:
+                getattr(self, generics_evaluator)(
+                    injected=injected_flight,
+                    sp_observed=None,
+                    dp_observed=None,
+                    participant=[],
+                    query_timestamp=datetime.datetime.now(),
+                )
+            except NoObservedFlight:
+                continue
+        for generics_evaluator in self.details_evaluators:
+            try:
+                getattr(self, generics_evaluator)(
+                    injected=injected_flight_details,
+                    sp_observed=None,
+                    dp_observed=None,
+                    participant=[],
+                    query_timestamp=datetime.datetime.now(),
+                )
+            except NoObservedFlight:
+                continue
+        for generics_evaluator in self.telemetry_evaluators:
+            try:
+                getattr(self, generics_evaluator)(
+                    injected=injected_telemetry,
+                    sp_observed=None,
+                    dp_observed=None,
+                    participant=[],
+                    query_timestamp=datetime.datetime.now(),
+                )
+            except NoObservedFlight:
+                continue
+
     def evaluate_sp_flight(
         self,
         injected_telemetry: injection.RIDAircraftState,
@@ -235,14 +278,14 @@ def evaluate_dp_details(
 
     def _evaluate_uas_id(
         self,
-        injected: injection.RIDFlightDetails,  # unused but required by callers
+        injected: injection.RIDFlightDetails,
         sp_observed: FlightDetails | None,
         dp_observed: observation_api.GetDetailsResponse | None,
         participant: ParticipantID,
         query_timestamp: datetime.datetime,
     ):
         """
-        Evaluates UAS id Exactly one of sp_observed or dp_observed must be provided.
+        Evaluates UAS id Exactly one of sp_observed or dp_observed must be provided, if not, will only evaluate injected values.
         See as well `common_dictionary_evaluator.md`.
 
         Raises:
@@ -253,6 +296,24 @@ def _evaluate_uas_id(
             # skip if evaluating DP: the UAS ID may be None, and if present is evaluated by evaluators specific to UAS ID types
             return
 
+        if sp_observed is None:
+            injected_values = []
+
+            for field in [
+                "uas_id.specific_session_id",
+                "uas_id.serial_number",
+                "uas_id.registration_id",
+                "uas_id.utm_id",
+            ]:
+                injected_values.append(_dotted_get(injected, field))
+
+            if not any(injected_values):
+                raise ValueError(
+                    "No valid UAS ID provided"
+                )  # NB: We may enounter invalid data for f3411_19, as a smaller subset of fields is needed, but we don't know how flight is going to be injected.
+
+            return NoObservedFlight("No observed flight")
+
         # We check that there is at least one value set
         with self._test_scenario.check(
             "UAS ID is exposed correctly", participant
@@ -304,7 +365,9 @@ def value_validator(val: SerialNumber) -> SerialNumber:
             serial_number = SerialNumber(val)
 
             if not serial_number.valid:
-                raise ValueError("Invalid serial number")
+                raise ValueError(
+                    f"Invalid serial number {SerialNumber.generate_valid()}"
+                )
 
             return serial_number
 
@@ -1225,7 +1288,7 @@ def _evaluate_ua_classification(
             ) and dp_observed.uas.has_field_with_value("eu_classification"):
                 observed_ua_classification = "eu_classification"
         else:
-            raise ValueError("No observed flight provided.")
+            raise NoObservedFlight("No observed flight provided.")
 
         with self._test_scenario.check(
             "UA classification type is consistent with injected value",
@@ -1411,7 +1474,13 @@ def _generic_evaluator(
         elif dp_observed is not None:
             observed_val = _dotted_get(dp_observed, dp_field_name)
         else:
-            raise ValueError("No observed flight provided.")
+            # Check that injected value is valid if required
+            if injected_val is None and injection_required_field:
+                raise ValueError(
+                    f"Field {field_human_name} is not defined, but is requiered."
+                )
+
+            raise NoObservedFlight("No observed flight provided.")
 
         if skip_eval:
             skip_reason = skip_eval(injected_val, observed_val)