Skip to content

Commit 95ed19e

Browse files
martinboulaismartinboulais
committed
WIP
1 parent 65fa6a2 commit 95ed19e

14 files changed

Lines changed: 287 additions & 241 deletions

File tree

‎lib/database/adapters/QcFlagAdapter.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ class QcFlagAdapter {
3030
/**
3131
* Converts the given database object to an entity object.
3232
*
33-
* @param {SequelizeQualityControlFlag} databaseObject Object to convert.
33+
* @param {SequelizeQcFlag} databaseObject Object to convert.
3434
* @returns {QcFlag} Converted entity object.
3535
*/
3636
toEntity(databaseObject) {

‎lib/database/adapters/RunAdapter.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,8 @@ class RunAdapter {
115115
userIdO2Stop,
116116
startTime,
117117
endTime,
118+
qcTimeStart,
119+
qcTimeEnd,
118120
runDuration,
119121
tags,
120122
updatedAt,
@@ -183,6 +185,8 @@ class RunAdapter {
183185
userIdO2Stop,
184186
startTime,
185187
endTime,
188+
qcTimeStart,
189+
qcTimeEnd,
186190
runDuration,
187191
environmentId,
188192
updatedAt: new Date(updatedAt).getTime(),
@@ -298,6 +302,8 @@ class RunAdapter {
298302
userIdO2Stop: entityObject.userIdO2Stop,
299303
startTime: entityObject.startTime,
300304
endTime: entityObject.endTime,
305+
qcTimeStart: entityObject.qcTimeStart,
306+
qcTimeEnd: entityObject.qcTimeEnd,
301307
environmentId: entityObject.environmentId,
302308
runTypeId: entityObject.runTypeId,
303309
runQuality: entityObject.runQuality,

‎lib/database/models/run.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,12 @@ module.exports = (sequelize) => {
6565
return (runEndString ? new Date(runEndString) : new Date()).getTime();
6666
},
6767
},
68+
qcTimeStart: {
69+
type: Sequelize.DATE,
70+
},
71+
qcTimeEnd: {
72+
type: Sequelize.DATE,
73+
},
6874
runDuration: {
6975
type: Sequelize.VIRTUAL,
7076
// eslint-disable-next-line require-jsdoc

‎lib/database/models/typedefs/SequelizeQcFlag.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616
*
1717
* @property {number} id
1818
* @property {boolean} deleted
19-
* @property {number} from
20-
* @property {number} to
19+
* @property {number|null} from
20+
* @property {number|null} to
2121
* @property {string} comment
2222
* @property {string} origin
2323
* @property {number} createdById

‎lib/database/models/typedefs/SequelizeRun.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
* @property {number|null} userIdO2Stop relation to the user id that stopped the run
2626
* @property {number|null} startTime timestamp of the run's start, either trigger start if it exists or o2 start or null
2727
* @property {number|null} endTime timestamp of the run's end, either trigger end if it exists or o2 end or now (always null if start is null)
28+
* @property {number|null} qcTimeStart coalesce of run first TF timestamp, trigger start and run o2 start
29+
* @property {number|null} qcTimeEnd coalesce of run last TF timestamp, trigger stop and run o2 end
2830
* @property {string|null} environmentId
2931
* @property {string} runQuality
3032
* @property {number|null} nDetectors

‎lib/database/repositories/QcFlagEffectivePeriodRepository.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,14 @@ class QcFlagEffectivePeriodRepository extends Repository {
3232
*
3333
* @param {Period} period period which effective periods must overlap with
3434
* @param {number|Date} createdAtUpperLimit upper limit of QC flags creation timestamp which effective periods are to be found
35+
* @param {object} monalisaProduction the scope of the flag
3536
* @param {number} [monalisaProduction.dataPassId] id of data pass, which the QC flag belongs to
3637
* @param {number} [monalisaProduction.simulationPassId] id of simulation pass, which the QC flags belongs to
3738
* @param {number} monalisaProduction.runNumber runNumber of run, which the QC flags belongs to
3839
* @param {number} monalisaProduction.detectorId id of DPL detector, which the QC flags belongs to
3940
* @return {Promise<SequelizeQcFlagEffectivePeriod[]>} effective periods promise
4041
*/
41-
async findOverlappingPeriodsCreatedNotAfter(period, createdAtUpperLimit, { dataPassId, simulationPassId, runNumber, detectorId }) {
42+
async findOverlappingPeriodsCreatedBeforeLimit(period, createdAtUpperLimit, { dataPassId, simulationPassId, runNumber, detectorId }) {
4243
const { to, from } = period;
4344

4445
const flagIncludes = [];

‎lib/database/repositories/QcFlagRepository.js‎

Lines changed: 33 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -16,23 +16,22 @@ const { models: { QcFlag } } = require('..');
1616
const Repository = require('./Repository');
1717

1818
const GAQ_PERIODS_VIEW = `
19+
SELECT * FROM (
1920
SELECT
2021
data_pass_id,
2122
run_number,
22-
timestamp AS \`from\`,
23-
NTH_VALUE(timestamp, 2) OVER (
24-
PARTITION BY data_pass_id,
25-
run_number
26-
ORDER BY ap.timestamp
27-
ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING
28-
) AS \`to\`
23+
LAG(timestamp) OVER w AS \`from\`,
24+
timestamp AS \`to\`,
25+
LAG(ordering_timestamp) OVER w AS from_ordering_timestamp
2926
FROM (
3027
(
3128
SELECT gaqd.data_pass_id,
3229
gaqd.run_number,
33-
COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS timestamp
30+
qcfep.\`from\` AS timestamp,
31+
COALESCE(qcfep.\`from\`, r.qc_time_start, '0001-01-01 00:00:00.000') AS ordering_timestamp
3432
FROM quality_control_flag_effective_periods AS qcfep
3533
INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id
34+
INNER JOIN runs AS r ON qcf.run_number = r.run_number
3635
INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
3736
-- Only flags of detectors which are defined in global_aggregated_quality_detectors
3837
-- should be taken into account for calculation of gaq_effective_periods
@@ -45,9 +44,11 @@ const GAQ_PERIODS_VIEW = `
4544
(
4645
SELECT gaqd.data_pass_id,
4746
gaqd.run_number,
48-
UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW())) AS timestamp
47+
qcfep.\`to\` AS timestamp,
48+
COALESCE(qcfep.\`to\`, r.qc_time_end, NOW()) AS ordering_timestamp
4949
FROM quality_control_flag_effective_periods AS qcfep
5050
INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id
51+
INNER JOIN runs AS r ON qcf.run_number = r.run_number
5152
INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
5253
-- Only flags of detectors which are defined in global_aggregated_quality_detectors
5354
-- should be taken into account for calculation of gaq_effective_periods
@@ -56,8 +57,15 @@ const GAQ_PERIODS_VIEW = `
5657
AND gaqd.run_number = qcf.run_number
5758
AND gaqd.detector_id = qcf.detector_id
5859
)
59-
ORDER BY timestamp
60-
) AS ap
60+
ORDER BY ordering_timestamp
61+
) AS ap
62+
WINDOW w AS (
63+
PARTITION BY data_pass_id,
64+
run_number
65+
ORDER BY ap.ordering_timestamp
66+
)
67+
) as gaq_periods_with_last_nullish_row
68+
WHERE gaq_periods_with_last_nullish_row.from_ordering_timestamp IS NOT NULL
6169
`;
6270

6371
/**
@@ -104,8 +112,8 @@ class QcFlagRepository extends Repository {
104112
SELECT
105113
gaq_periods.data_pass_id AS dataPassId,
106114
gaq_periods.run_number AS runNumber,
107-
IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\` * 1000) AS \`from\`,
108-
IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\` * 1000) AS \`to\`,
115+
gaq_periods.\`from\` AS \`from\`,
116+
gaq_periods.\`to\` AS \`to\`,
109117
group_concat(qcf.id) AS contributingFlagIds
110118
111119
FROM quality_control_flags AS qcf
@@ -118,8 +126,8 @@ class QcFlagRepository extends Repository {
118126
AND gaqd.run_number = gaq_periods.run_number
119127
AND gaqd.detector_id = qcf.detector_id
120128
AND gaq_periods.run_number = qcf.run_number
121-
AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`)
122-
AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`))
129+
AND (qcfep.\`from\` IS NULL OR qcfep.\`from\` <= gaq_periods.\`from\`)
130+
AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= qcfep.\`to\`)
123131
124132
WHERE gaq_periods.data_pass_id = ${dataPassId}
125133
${runNumber ? `AND gaq_periods.run_number = ${runNumber}` : ''}
@@ -140,8 +148,8 @@ class QcFlagRepository extends Repository {
140148
}) => ({
141149
dataPassId,
142150
runNumber,
143-
from,
144-
to,
151+
from: from?.getTime(),
152+
to: to?.getTime(),
145153
contributingFlagIds: contributingFlagIds.split(',').map((id) => parseInt(id, 10)),
146154
}));
147155
}
@@ -160,8 +168,8 @@ class QcFlagRepository extends Repository {
160168
SELECT
161169
gaq_periods.data_pass_id AS dataPassId,
162170
gaq_periods.run_number AS runNumber,
163-
IF(gaq_periods.\`from\` = 0, null, gaq_periods.\`from\`) AS \`from\`,
164-
IF(gaq_periods.\`to\` = UNIX_TIMESTAMP(NOW()), null, gaq_periods.\`to\`) AS \`to\`,
171+
gaq_periods.\`from\` AS \`from\`,
172+
gaq_periods.\`to\` AS \`to\`,
165173
SUM(IF(qcft.monte_carlo_reproducible AND :mcReproducibleAsNotBad, false, qcft.bad)) >= 1 AS bad,
166174
SUM(qcft.bad) = SUM(qcft.monte_carlo_reproducible) AND SUM(qcft.monte_carlo_reproducible) AS mcReproducible,
167175
GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verifiedFlagsList,
@@ -183,8 +191,8 @@ class QcFlagRepository extends Repository {
183191
AND gaqd.run_number = gaq_periods.run_number
184192
AND gaqd.detector_id = qcf.detector_id
185193
AND gaq_periods.run_number = qcf.run_number
186-
AND (qcfep.\`from\` IS NULL OR UNIX_TIMESTAMP(qcfep.\`from\`) <= gaq_periods.\`from\`)
187-
AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= UNIX_TIMESTAMP(qcfep.\`to\`))
194+
AND (qcfep.\`from\` IS NULL OR qcfep.\`from\` <= gaq_periods.\`from\`)
195+
AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= qcfep.\`to\`)
188196
189197
GROUP BY
190198
gaq_periods.data_pass_id,
@@ -203,18 +211,16 @@ class QcFlagRepository extends Repository {
203211
GROUP_CONCAT(effectivePeriods.flagsList) AS flagsList,
204212
205213
IF(
206-
run.time_start IS NULL OR run.time_end IS NULL,
214+
run.qc_time_start IS NULL OR run.qc_time_end IS NULL,
207215
IF(
208216
effectivePeriods.\`from\` IS NULL AND effectivePeriods.\`to\` IS NULL,
209217
1,
210218
null
211219
),
212220
SUM(
213-
COALESCE(effectivePeriods.\`to\`, UNIX_TIMESTAMP(run.time_end))
214-
- COALESCE(effectivePeriods.\`from\`, UNIX_TIMESTAMP(run.time_start))
215-
) / (
216-
UNIX_TIMESTAMP(run.time_end) - UNIX_TIMESTAMP(run.time_start)
217-
)
221+
UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`to\`,run.qc_time_end))
222+
- UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`from\`, run.qc_time_start))
223+
) / (UNIX_TIMESTAMP(run.qc_time_end) - UNIX_TIMESTAMP(run.qc_time_start))
218224
) AS effectiveRunCoverage
219225
220226
FROM (${effectivePeriodsWithTypeSubQuery}) AS effectivePeriods

‎lib/domain/entities/QcFlag.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616
*
1717
* @property {number} id
1818
* @property {boolean} deleted
19-
* @property {number} from
20-
* @property {number} to
19+
* @property {number|null} from
20+
* @property {number|null} to
2121
* @property {string} comment
2222
* @property {string} origin
2323
* @property {number} createdById

‎lib/domain/entities/Run.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
* @property {number|null} userIdO2Stop relation to the user id that stopped the run
2626
* @property {Date|null} startTime timestamp of the run's start, either trigger start if it exists or o2 start or null
2727
* @property {Date|null} endTime timestamp of the run's end, either trigger end if it exists or o2 end or now (always null if start is null)
28+
* @property {Date|null} qcTimeStart coalesce of run first TF timestamp, trigger start and run o2 start
29+
* @property {Date|null} qcTimeEnd coalesce of run last TF timestamp, trigger stop and run o2 end
2830
* @property {string|null} environmentId
2931
* @property {number|null} runTypeId
3032
* @property {string} runQuality

‎lib/server/services/qualityControlFlag/GaqService.js‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,27 @@ import { qcFlagAdapter } from '../../../database/adapters/index.js';
1616
import { QcFlagRepository } from '../../../database/repositories/index.js';
1717
import { QcFlagSummaryService } from './QcFlagSummaryService.js';
1818

19+
/**
20+
* @typedef GaqFlags
21+
*
22+
* @property {number} from
23+
* @property {number} to
24+
* @property {QcFlag[]} contributingFlags
25+
*/
26+
27+
/**
28+
* @typedef RunGaqSummary
29+
* @property {number} badEffectiveRunCoverage - fraction of run's data, which aggregated quality is bad
30+
* @property {number} explicitlyNotBadEffectiveRunCoverage - fraction of run's data, which aggregated quality is explicitly good
31+
* @property {number} missingVerificationsCount - number of not verified QC flags which are not discarded
32+
* @property {boolean} mcReproducible - states whether some aggregation of QC flags is Limited Acceptance MC Reproducible
33+
*/
34+
35+
/**
36+
* @typedef GaqSummary aggregated global quality summaries for given data pass
37+
* @type {Object<number, RunGaqSummary>} runNumber to RunGaqSummary mapping
38+
*/
39+
1940
const QC_SUMMARY_PROPERTIES = {
2041
badEffectiveRunCoverage: 'badEffectiveRunCoverage',
2142
explicitlyNotBadEffectiveRunCoverage: 'explicitlyNotBadEffectiveRunCoverage',

0 commit comments

Comments
 (0)