[O2B-1435] Rework QC flags storage to have nullable bounds (#1871)

* Extract dedicated services for GAQ and QC summaries

* Update seeders so run 106 do not have first TF timestamp

* WIP

* WIP on tests

* Finalize service test and fix linter

* Fix more tests

* Minor refactoring and documentation rewrite

* Improve naming

* Add tests

* Try to fix balloon failing test

* Fix linter

* Another attempt at fixing balloon tests

* Fix imports

* Fix missing import

* Restrict effective run coverage between 0 and 1

* Add missing coalesce

* Fix tests again

* Add migration to set null boundaries

Author: martinboulais

lib/database/adapters/QcFlagAdapter.js

@@ -30,7 +30,7 @@ class QcFlagAdapter {
     /**
      * Converts the given database object to an entity object.
      *
-     * @param {SequelizeQualityControlFlag} databaseObject Object to convert.
+     * @param {SequelizeQcFlag} databaseObject Object to convert.
      * @returns {QcFlag} Converted entity object.
      */
     toEntity(databaseObject) {

lib/database/adapters/RunAdapter.js

@@ -115,6 +115,8 @@ class RunAdapter {
             userIdO2Stop,
             startTime,
             endTime,
+            qcTimeStart,
+            qcTimeEnd,
             runDuration,
             tags,
             updatedAt,
@@ -183,6 +185,8 @@ class RunAdapter {
             userIdO2Stop,
             startTime,
             endTime,
+            qcTimeStart,
+            qcTimeEnd,
             runDuration,
             environmentId,
             updatedAt: new Date(updatedAt).getTime(),
@@ -298,6 +302,8 @@ class RunAdapter {
             userIdO2Stop: entityObject.userIdO2Stop,
             startTime: entityObject.startTime,
             endTime: entityObject.endTime,
+            qcTimeStart: entityObject.qcTimeStart,
+            qcTimeEnd: entityObject.qcTimeEnd,
             environmentId: entityObject.environmentId,
             runTypeId: entityObject.runTypeId,
             runQuality: entityObject.runQuality,

lib/database/migrations/v1/20250326133702-set-flag-bound-null-when-equal-to-run-end.js

@@ -0,0 +1,56 @@
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+    up: async (queryInterface) => queryInterface.sequelize.transaction(async () => {
+        await queryInterface.sequelize.query(`UPDATE quality_control_flags qcf INNER JOIN runs r ON qcf.run_number = r.run_number
+                                              SET qcf.\`from\` = NULL
+                                              WHERE r.qc_time_start IS NOT NULL
+                                                AND qcf.\`from\` = r.qc_time_start`);
+
+        await queryInterface.sequelize.query(`UPDATE quality_control_flag_effective_periods qcfep
+                                              INNER JOIN quality_control_flags qcf ON qcfep.flag_id = qcf.id
+                                              INNER JOIN runs r ON qcf.run_number = r.run_number
+                                              SET qcfep.\`from\` = NULL
+                                              WHERE r.qc_time_start IS NOT NULL
+                                                AND qcfep.\`from\` = r.qc_time_start`);
+
+        await queryInterface.sequelize.query(`UPDATE quality_control_flags qcf INNER JOIN runs r ON qcf.run_number = r.run_number
+                                              SET qcf.\`to\` = NULL
+                                              WHERE r.qc_time_end IS NOT NULL
+                                                AND qcf.\`to\` = r.qc_time_end`);
+
+        await queryInterface.sequelize.query(`UPDATE quality_control_flag_effective_periods qcfep 
+                                              INNER JOIN quality_control_flags qcf ON qcfep.flag_id = qcf.id
+                                              INNER JOIN runs r ON qcf.run_number = r.run_number
+                                              SET qcfep.\`to\` = NULL
+                                              WHERE r.qc_time_end IS NOT NULL
+                                                AND qcfep.\`to\` = r.qc_time_end`);
+    }),
+
+    down: async (queryInterface) => queryInterface.sequelize.transaction(async () => {
+        await queryInterface.sequelize.query(`UPDATE quality_control_flags qcf INNER JOIN runs r ON qcf.run_number = r.run_number
+                                              SET qcf.\`from\` = r.qc_time_start
+                                              WHERE r.qc_time_start IS NOT NULL
+                                                AND qcf.\`from\` IS NULL`);
+
+        await queryInterface.sequelize.query(`UPDATE quality_control_flag_effective_periods qcfep
+                                              INNER JOIN quality_control_flags qcf ON qcfep.flag_id = qcf.id
+                                              INNER JOIN runs r ON qcf.run_number = r.run_number
+                                              SET qcfep.\`from\` = r.qc_time_start
+                                              WHERE r.qc_time_start IS NOT NULL
+                                                AND qcfep.\`from\` IS NULL`);
+
+        await queryInterface.sequelize.query(`UPDATE quality_control_flags qcf INNER JOIN runs r ON qcf.run_number = r.run_number
+                                              SET qcf.\`to\` = r.qc_time_end
+                                              WHERE r.qc_time_end IS NOT NULL
+                                                AND qcf.\`to\` IS NULL`);
+
+        await queryInterface.sequelize.query(`UPDATE quality_control_flag_effective_periods qcfep 
+                                              INNER JOIN quality_control_flags qcf ON qcfep.flag_id = qcf.id
+                                              INNER JOIN runs r ON qcf.run_number = r.run_number
+                                              SET qcfep.\`to\` = r.qc_time_end
+                                              WHERE r.qc_time_end IS NOT NULL
+                                                AND qcfep.\`to\` IS NULL`);
+    }),
+};

lib/database/models/run.js

@@ -65,6 +65,12 @@ module.exports = (sequelize) => {
                 return (runEndString ? new Date(runEndString) : new Date()).getTime();
             },
         },
+        qcTimeStart: {
+            type: Sequelize.DATE,
+        },
+        qcTimeEnd: {
+            type: Sequelize.DATE,
+        },
         runDuration: {
             type: Sequelize.VIRTUAL,
             // eslint-disable-next-line require-jsdoc

lib/database/models/typedefs/SequelizeQcFlag.js

@@ -16,8 +16,8 @@
  *
  * @property {number} id
  * @property {boolean} deleted
- * @property {number} from
- * @property {number} to
+ * @property {number|null} from
+ * @property {number|null} to
  * @property {string} comment
  * @property {string} origin
  * @property {number} createdById

lib/database/models/typedefs/SequelizeRun.js

@@ -25,6 +25,8 @@
  * @property {number|null} userIdO2Stop relation to the user id that stopped the run
  * @property {number|null} startTime timestamp of the run's start, either trigger start if it exists or o2 start or null
  * @property {number|null} endTime timestamp of the run's end, either trigger end if it exists or o2 end or now (always null if start is null)
+ * @property {number|null} qcTimeStart coalesce of run first TF timestamp, trigger start and run o2 start
+ * @property {number|null} qcTimeEnd coalesce of run last TF timestamp, trigger stop and run o2 end
  * @property {string|null} environmentId
  * @property {string} runQuality
  * @property {number|null} nDetectors

lib/database/repositories/QcFlagEffectivePeriodRepository.js

@@ -32,13 +32,14 @@ class QcFlagEffectivePeriodRepository extends Repository {
      *
      * @param {Period} period period which effective periods must overlap with
      * @param {number|Date} createdAtUpperLimit upper limit of QC flags creation timestamp which effective periods are to be found
+     * @param {object} monalisaProduction the scope of the flag
      * @param {number} [monalisaProduction.dataPassId] id of data pass, which the QC flag belongs to
      * @param {number} [monalisaProduction.simulationPassId] id of simulation pass, which the QC flags belongs to
      * @param {number} monalisaProduction.runNumber runNumber of run, which the QC flags belongs to
      * @param {number} monalisaProduction.detectorId id of DPL detector, which the QC flags belongs to
      * @return {Promise<SequelizeQcFlagEffectivePeriod[]>} effective periods promise
      */
-    async findOverlappingPeriodsCreatedNotAfter(period, createdAtUpperLimit, { dataPassId, simulationPassId, runNumber, detectorId }) {
+    async findOverlappingPeriodsCreatedBeforeLimit(period, createdAtUpperLimit, { dataPassId, simulationPassId, runNumber, detectorId }) {
         const { to, from } = period;
 
         const flagIncludes = [];

lib/database/repositories/QcFlagRepository.js

@@ -16,23 +16,22 @@ const { models: { QcFlag } } = require('..');
 const Repository = require('./Repository');
 
 const GAQ_PERIODS_VIEW = `
+    SELECT * FROM (
         SELECT
             data_pass_id,
             run_number,
-            timestamp AS \`from\`,
-            NTH_VALUE(timestamp, 2) OVER (
-                PARTITION BY data_pass_id,
-                run_number
-                ORDER BY ap.timestamp
-                ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING
-            ) AS \`to\`
+            LAG(timestamp) OVER w AS \`from\`,
+            timestamp AS \`to\`,
+            LAG(ordering_timestamp) OVER w AS from_ordering_timestamp
         FROM (
                 (
                     SELECT gaqd.data_pass_id,
                         gaqd.run_number,
-                        COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS timestamp
+                        COALESCE(qcfep.\`from\`, r.qc_time_start) AS timestamp,
+                        COALESCE(qcfep.\`from\`, r.qc_time_start, '0001-01-01 00:00:00.000') AS ordering_timestamp
                     FROM quality_control_flag_effective_periods AS qcfep
                         INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id
+                        INNER JOIN runs AS r ON qcf.run_number = r.run_number
                         INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
                         -- Only flags of detectors which are defined in global_aggregated_quality_detectors
                         -- should be taken into account for calculation of gaq_effective_periods
@@ -45,9 +44,11 @@ const GAQ_PERIODS_VIEW = `
                 (
                     SELECT gaqd.data_pass_id,
                         gaqd.run_number,
-                        UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW())) AS timestamp
+                        COALESCE(qcfep.\`to\`, r.qc_time_end) AS timestamp,
+                        COALESCE(qcfep.\`to\`, r.qc_time_end, NOW()) AS ordering_timestamp
                     FROM quality_control_flag_effective_periods AS qcfep
                         INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id
+                        INNER JOIN runs AS r ON qcf.run_number = r.run_number
                         INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
                         -- Only flags of detectors which are defined in global_aggregated_quality_detectors
                         -- should be taken into account for calculation of gaq_effective_periods
@@ -56,8 +57,15 @@ const GAQ_PERIODS_VIEW = `
                             AND gaqd.run_number = qcf.run_number
                             AND gaqd.detector_id = qcf.detector_id
                 )
-                ORDER BY timestamp
-            ) AS ap
+                ORDER BY ordering_timestamp
+        ) AS ap
+        WINDOW w AS (
+            PARTITION BY data_pass_id,
+            run_number
+            ORDER BY ap.ordering_timestamp
+        )
+    ) as gaq_periods_with_last_nullish_row
+    WHERE gaq_periods_with_last_nullish_row.from_ordering_timestamp IS NOT NULL
     `;
 
 /**
@@ -104,8 +112,8 @@ class QcFlagRepository extends Repository {
             SELECT
                 gaq_periods.data_pass_id AS dataPassId,
                 gaq_periods.run_number AS runNumber,
-                IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\` * 1000) AS \`from\`,
-                IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\` * 1000) AS \`to\`,
+                gaq_periods.\`from\` AS \`from\`,
+                gaq_periods.\`to\` AS \`to\`,
                 group_concat(qcf.id) AS contributingFlagIds
 
             FROM quality_control_flags AS qcf
@@ -118,8 +126,8 @@ class QcFlagRepository extends Repository {
                     AND gaqd.run_number = gaq_periods.run_number
                     AND gaqd.detector_id = qcf.detector_id
                     AND gaq_periods.run_number = qcf.run_number
-                        AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`)
-                        AND (qcfep.\`to\`   IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`))
+                        AND (qcfep.\`from\` IS NULL OR qcfep.\`from\` <= gaq_periods.\`from\`)
+                        AND (qcfep.\`to\`   IS NULL OR gaq_periods.\`to\` <= qcfep.\`to\`)
     
             WHERE gaq_periods.data_pass_id = ${dataPassId}
                 ${runNumber ? `AND gaq_periods.run_number = ${runNumber}` : ''}
@@ -140,8 +148,8 @@ class QcFlagRepository extends Repository {
         }) => ({
             dataPassId,
             runNumber,
-            from,
-            to,
+            from: from?.getTime(),
+            to: to?.getTime(),
             contributingFlagIds: contributingFlagIds.split(',').map((id) => parseInt(id, 10)),
         }));
     }
@@ -160,8 +168,8 @@ class QcFlagRepository extends Repository {
             SELECT
                 gaq_periods.data_pass_id AS dataPassId,
                 gaq_periods.run_number AS runNumber,
-                IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\`) AS \`from\`,
-                IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) AS \`to\`,
+                gaq_periods.\`from\` AS \`from\`,
+                gaq_periods.\`to\` AS \`to\`,
                 SUM(IF(qcft.monte_carlo_reproducible AND :mcReproducibleAsNotBad, false, qcft.bad)) >= 1 AS bad,
                 SUM(qcft.bad) = SUM(qcft.monte_carlo_reproducible) AND SUM(qcft.monte_carlo_reproducible) AS mcReproducible,
                 GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verifiedFlagsList,
@@ -183,8 +191,8 @@ class QcFlagRepository extends Repository {
                     AND gaqd.run_number = gaq_periods.run_number
                     AND gaqd.detector_id = qcf.detector_id
                     AND gaq_periods.run_number = qcf.run_number
-                        AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`)
-                        AND (qcfep.\`to\`   IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`))
+                        AND (qcfep.\`from\` IS NULL OR qcfep.\`from\` <= gaq_periods.\`from\`)
+                        AND (qcfep.\`to\`   IS NULL OR gaq_periods.\`to\` <= qcfep.\`to\`)
 
             GROUP BY
                 gaq_periods.data_pass_id,
@@ -203,18 +211,16 @@ class QcFlagRepository extends Repository {
                 GROUP_CONCAT(effectivePeriods.flagsList) AS flagsList,
 
                 IF(
-                    run.time_start IS NULL OR run.time_end IS NULL,
+                    run.qc_time_start IS NULL OR run.qc_time_end IS NULL,
                     IF(
                         effectivePeriods.\`from\` IS NULL AND effectivePeriods.\`to\`  IS NULL,
                         1,
                         null
                     ),
                     SUM(
-                        COALESCE(effectivePeriods.\`to\`, UNIX_TIMESTAMP(run.time_end))
-                      - COALESCE(effectivePeriods.\`from\`, UNIX_TIMESTAMP(run.time_start))
-                    ) / (
-                        UNIX_TIMESTAMP(run.time_end) - UNIX_TIMESTAMP(run.time_start)
-                    )
+                        UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`to\`,run.qc_time_end))
+                        - UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`from\`, run.qc_time_start))
+                    ) / (UNIX_TIMESTAMP(run.qc_time_end) - UNIX_TIMESTAMP(run.qc_time_start))
                 ) AS effectiveRunCoverage
 
             FROM (${effectivePeriodsWithTypeSubQuery}) AS effectivePeriods
@@ -236,14 +242,20 @@ class QcFlagRepository extends Repository {
             mcReproducible,
             flagsList,
             verifiedFlagsList,
-        }) => ({
-            runNumber,
-            bad,
-            effectiveRunCoverage: parseFloat(effectiveRunCoverage) || null,
-            mcReproducible: Boolean(mcReproducible),
-            flagsIds: [...new Set(flagsList.split(','))],
-            verifiedFlagsIds: verifiedFlagsList ? [...new Set(verifiedFlagsList.split(','))] : [],
-        }));
+        }) => {
+            if ((effectiveRunCoverage ?? null) != null) {
+                effectiveRunCoverage = Math.min(1, Math.max(0, parseFloat(effectiveRunCoverage)));
+            }
+
+            return {
+                runNumber,
+                bad,
+                effectiveRunCoverage,
+                mcReproducible: Boolean(mcReproducible),
+                flagsIds: [...new Set(flagsList.split(','))],
+                verifiedFlagsIds: verifiedFlagsList ? [...new Set(verifiedFlagsList.split(','))] : [],
+            };
+        });
     }
 
     /**

lib/database/seeders/20200713103855-runs.js

@@ -2509,6 +2509,8 @@ module.exports = {
                 {
                     id: 100,
                     run_number: 100,
+                    time_o2_end: '2019-08-08 13:00:00',
+                    time_trg_end: '2019-08-08 14:23:45',
                     run_type_id: 12,
                     run_quality: 'test',
                     n_detectors: 3,
@@ -2661,7 +2663,6 @@ module.exports = {
                     time_o2_end: '2019-08-09 14:00:00',
                     time_trg_start: '2019-08-08 13:00:00',
                     time_trg_end: '2019-08-09 14:00:00',
-                    first_tf_timestamp: '2019-08-09 13:00:00',
                     run_type_id: 12,
                     run_quality: 'good',
                     n_detectors: 15,
@@ -2737,9 +2738,10 @@ module.exports = {
                     id: 108,
                     run_number: 108,
                     time_o2_start: '2019-08-08 13:00:00',
-                    time_o2_end: '2019-08-09 14:00:00',
+                    first_tf_timestamp: '2019-08-08 13:00:00',
                     time_trg_start: '2019-08-08 13:00:00',
                     time_trg_end: '2019-08-09 14:00:00',
+                    time_o2_end: '2019-08-09 14:00:00',
                     run_type_id: 12,
                     run_quality: 'good',
                     n_detectors: 15,

lib/database/seeders/20240112102011-data-passes.js

@@ -203,6 +203,7 @@ module.exports = {
                 { data_pass_id: 1, run_number: 106, ready_for_skimming: true },
                 { data_pass_id: 1, run_number: 107, ready_for_skimming: false },
                 { data_pass_id: 1, run_number: 108, ready_for_skimming: false },
+
                 { data_pass_id: 2, run_number: 1 },
                 { data_pass_id: 2, run_number: 2 },
                 { data_pass_id: 2, run_number: 55 },
@@ -215,6 +216,7 @@ module.exports = {
                 { data_pass_id: 4, run_number: 49 },
                 { data_pass_id: 4, run_number: 54 },
                 { data_pass_id: 4, run_number: 56 },
+                { data_pass_id: 4, run_number: 100 },
                 { data_pass_id: 4, run_number: 105 },
 
                 { data_pass_id: 5, run_number: 49 },