Skip to content

Commit df0fc06

Browse files
martinboulaismartinboulais
authored
Merge branch 'main' into mboulais/attempt-at-fixing-eos-random-tests
2 parents ef94d8c + f4b1329 commit df0fc06

73 files changed

Lines changed: 1537 additions & 720 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CHANGELOG.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,24 @@
22

33
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
44

5+
## [0.91.0](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%400.91.0)
6+
* Notable changes for users:
7+
* Added a new tab in runs details to display trigger counters table
8+
* Disable sorting reconstructedEventsCount and outputSize on DataPasses overview pages
9+
* Added iconBan to RCT runs overviews in case there is no QC flag assigned to detector and user has no permission to add one
10+
* Added gRPC interface to create/update trigger counters
11+
* Changed display of Bad runs in Runs per data/simulation pass views and disabled QC management for them
12+
* Notable change for developers:
13+
* Several improvements has been done on frontend suite:
14+
* Set default navigation timeout to 5s
15+
* Set default timeout (except for navigation) to 1.5s
16+
* Removed waitForNetworkIdleAndRedraw and improved tests incorrectly waiting for navigation
17+
* Replaced checkMismatchingUrlParam by easier to use function expectUrlParams
18+
* Removed function validateElement and replaced its calls by native waitForSelector
19+
* Refactored ML client to fetch data passes data n CSV format
20+
* Refactored ML synchronization to work with lastSeen instead of lastRunNumber
21+
* Factorized QC flags breadcrumbs
22+
523
## [0.90.0](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%400.90.0)
624
* Notable changes for users:
725
* Ordered magnets input Dipole first then solenoid in RC daily meeting template and SL EoS report

‎database/CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
## [0.91.0](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%400.91.0)
2+
* Changes made to the database
3+
* Added data_pass_versions table
4+
15
## [0.90.0](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%400.90.0)
26
* Changes made to the database
37
* Added origin column to quality_control_flags table

‎lib/database/adapters/DataPassAdapter.js‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ class DataPassAdapter {
2020
*/
2121
constructor() {
2222
this.toEntity = this.toEntity.bind(this);
23+
this.dataPassVersionAdapter = null;
2324
}
2425

2526
/**
@@ -32,10 +33,7 @@ class DataPassAdapter {
3233
const {
3334
id,
3435
name,
35-
description,
36-
outputSize,
37-
reconstructedEventsCount,
38-
lastRunNumber,
36+
versions,
3937
} = databaseObject;
4038

4139
const runsCount = databaseObject.get('runsCount');
@@ -44,10 +42,7 @@ class DataPassAdapter {
4442
return {
4543
id,
4644
name,
47-
description,
48-
outputSize,
49-
reconstructedEventsCount,
50-
lastRunNumber,
45+
versions: (versions ?? []).map(this.dataPassVersionAdapter.toEntity),
5146
runsCount,
5247
simulationPassesCount,
5348
};

‎lib/database/adapters/DataPassVersionAdapter.js‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/**
2+
* @license
3+
* Copyright CERN and copyright holders of ALICE O2. This software is
4+
* distributed under the terms of the GNU General Public License v3 (GPL
5+
* Version 3), copied verbatim in the file "COPYING".
6+
*
7+
* See http://alice-o2.web.cern.ch/license for full licensing information.
8+
*
9+
* In applying this license CERN does not waive the privileges and immunities
10+
* granted to it by virtue of its status as an Intergovernmental Organization
11+
* or submit itself to any jurisdiction.
12+
*/
13+
14+
/**
15+
* DataPassVersionAdapter
16+
*/
17+
class DataPassVersionAdapter {
18+
/**
19+
* Constructor
20+
*/
21+
constructor() {
22+
this.toEntity = this.toEntity.bind(this);
23+
}
24+
25+
/**
26+
* Converts the given database object to an entity object.
27+
*
28+
* @param {SequelizeDataPass} databaseObject Object to convert.
29+
* @returns {DataPass} Converted entity object.
30+
*/
31+
toEntity(databaseObject) {
32+
const {
33+
id,
34+
dataPassId,
35+
description,
36+
reconstructedEventsCount,
37+
outputSize,
38+
lastSeen,
39+
createdAt,
40+
updatedAt,
41+
} = databaseObject;
42+
43+
return {
44+
id,
45+
dataPassId,
46+
description,
47+
reconstructedEventsCount,
48+
outputSize,
49+
lastSeen,
50+
createdAt: createdAt ? new Date(createdAt).getTime() : null,
51+
updatedAt: updatedAt ? new Date(updatedAt).getTime() : null,
52+
};
53+
}
54+
}
55+
56+
module.exports = { DataPassVersionAdapter };

‎lib/database/adapters/index.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
const AttachmentAdapter = require('./AttachmentAdapter');
1515
const DataPassAdapter = require('./DataPassAdapter');
1616
const { DataPassQcFlagAdapter } = require('./DataPassQcFlagAdapter.js');
17+
const { DataPassVersionAdapter } = require('./DataPassVersionAdapter.js');
1718
const DetectorAdapter = require('./DetectorAdapter');
1819
const { DplDetectorAdapter } = require('./dpl/DplDetectorAdapter.js');
1920
const { DplProcessExecutionAdapter } = require('./dpl/DplProcessExecutionAdapter.js');
@@ -49,6 +50,7 @@ const UserAdapter = require('./UserAdapter');
4950
const attachmentAdapter = new AttachmentAdapter();
5051
const dataPassQcFlagAdapter = new DataPassQcFlagAdapter();
5152
const dataPassAdapter = new DataPassAdapter();
53+
const dataPassVersionAdapter = new DataPassVersionAdapter();
5254
const detectorAdapter = new DetectorAdapter();
5355
const dplDetectorAdapter = new DplDetectorAdapter();
5456
const dplProcessAdapter = new DplProcessAdapter();
@@ -82,6 +84,7 @@ const triggerCountersAdapter = new TriggerCountersAdapter();
8284
const userAdapter = new UserAdapter();
8385

8486
// Fill dependencies
87+
dataPassAdapter.dataPassVersionAdapter = dataPassVersionAdapter;
8588
dataPassQcFlagAdapter.dataPassAdapter = dataPassAdapter;
8689
dataPassQcFlagAdapter.qcFlagAdapter = qcFlagAdapter;
8790

@@ -141,6 +144,7 @@ module.exports = {
141144
attachmentAdapter,
142145
dataPassAdapter,
143146
dataPassQcFlagAdapter,
147+
dataPassVersionAdapter,
144148
detectorAdapter,
145149
dplDetectorAdapter,
146150
dplProcessAdapter,

‎lib/database/migrations/20240606104339-extract-data-passes-versions-table.js‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/**
2+
* @license
3+
* Copyright CERN and copyright holders of ALICE O2. This software is
4+
* distributed under the terms of the GNU General Public License v3 (GPL
5+
* Version 3), copied verbatim in the file "COPYING".
6+
*
7+
* See http://alice-o2.web.cern.ch/license for full licensing information.
8+
*
9+
* In applying this license CERN does not waive the privileges and immunities
10+
* granted to it by virtue of its status as an Intergovernmental Organization
11+
* or submit itself to any jurisdiction.
12+
*/
13+
14+
'use strict';
15+
16+
const { Sequelize } = require('sequelize');
17+
18+
/** @type {import('sequelize-cli').Migration} */
19+
module.exports = {
20+
up: async (queryInterface) => queryInterface.sequelize.transaction(async (transaction) => {
21+
await queryInterface.createTable('data_pass_versions', {
22+
id: {
23+
type: Sequelize.INTEGER,
24+
primaryKey: true,
25+
autoIncrement: true,
26+
allowNull: false,
27+
},
28+
data_pass_id: {
29+
type: Sequelize.INTEGER,
30+
allowNull: false,
31+
references: {
32+
model: 'data_passes',
33+
key: 'id',
34+
},
35+
},
36+
description: {
37+
type: Sequelize.STRING,
38+
unique: true,
39+
allowNull: false,
40+
},
41+
output_size: {
42+
type: Sequelize.BIGINT,
43+
},
44+
reconstructed_events_count: {
45+
type: Sequelize.INTEGER,
46+
},
47+
last_seen: {
48+
type: Sequelize.INTEGER,
49+
comment: 'Change of this property in MonALISA means recent job activity that changed data pass version properties',
50+
},
51+
52+
// Timestamps
53+
created_at: {
54+
allowNull: false,
55+
type: Sequelize.DATE,
56+
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP'),
57+
},
58+
updated_at: {
59+
allowNull: false,
60+
type: Sequelize.DATE,
61+
defaultValue: Sequelize.literal('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'),
62+
},
63+
}, { transaction });
64+
65+
// Migrate
66+
await queryInterface.sequelize.query(`
67+
INSERT INTO data_pass_versions (data_pass_id, description, output_size, reconstructed_events_count)
68+
SELECT id, description, output_size, reconstructed_events_count
69+
FROM data_passes
70+
`, { transaction });
71+
72+
await queryInterface.removeColumn('data_passes', 'description', { transaction });
73+
await queryInterface.removeColumn('data_passes', 'output_size', { transaction });
74+
await queryInterface.removeColumn('data_passes', 'reconstructed_events_count', { transaction });
75+
await queryInterface.removeColumn('data_passes', 'last_run_number', { transaction });
76+
}),
77+
78+
down: async (queryInterface) => queryInterface.sequelize.transaction(async (transaction) => {
79+
await queryInterface.addColumn('data_passes', 'description', { type: Sequelize.TEXT }, { transaction });
80+
await queryInterface.addColumn('data_passes', 'output_size', { type: Sequelize.BIGINT }, { transaction });
81+
await queryInterface.addColumn('data_passes', 'reconstructed_events_count', { type: Sequelize.INTEGER }, { transaction });
82+
await queryInterface.addColumn('data_passes', 'last_run_number', { type: Sequelize.INTEGER }, { transaction });
83+
84+
// Migrate
85+
await queryInterface.sequelize.query(`
86+
UPDATE data_passes
87+
INNER JOIN (
88+
WITH lastwin as (
89+
SELECT
90+
id,
91+
data_pass_id,
92+
description,
93+
output_size,
94+
reconstructed_events_count,
95+
ROW_NUMBER() OVER (PARTITION BY data_pass_id ORDER BY id DESC) as rn
96+
FROM data_pass_versions
97+
)
98+
SELECT data_pass_id, description, output_size, reconstructed_events_count
99+
FROM lastwin
100+
WHERE rn = 1
101+
) as recent_data_pass_version
102+
103+
ON recent_data_pass_version.data_pass_id = data_passes.id
104+
105+
SET data_passes.description = recent_data_pass_version.description,
106+
data_passes.output_size = recent_data_pass_version.output_size,
107+
data_passes.reconstructed_events_count = recent_data_pass_version.reconstructed_events_count
108+
109+
`, { transaction });
110+
111+
await queryInterface.dropTable('data_pass_versions', { transaction });
112+
}),
113+
};

‎lib/database/models/dataPass.js‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,18 +27,6 @@ module.exports = (sequelize) => {
2727
type: Sequelize.STRING,
2828
allowNull: false,
2929
},
30-
description: {
31-
type: Sequelize.TEXT,
32-
},
33-
outputSize: {
34-
type: Sequelize.BIGINT,
35-
},
36-
reconstructedEventsCount: {
37-
type: Sequelize.INTEGER,
38-
},
39-
lastRunNumber: {
40-
type: Sequelize.INTEGER,
41-
},
4230
},
4331
{ tableName: 'data_passes' },
4432
);
@@ -57,6 +45,7 @@ module.exports = (sequelize) => {
5745
foreignKey: 'data_pass_id',
5846
through: 'data_pass_quality_control_flag',
5947
});
48+
DataPass.hasMany(models.DataPassVersion, { as: 'versions' });
6049
};
6150

6251
return DataPass;

‎lib/database/models/dataPassVersion.js‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/**
2+
* @license
3+
* Copyright CERN and copyright holders of ALICE O2. This software is
4+
* distributed under the terms of the GNU General Public License v3 (GPL
5+
* Version 3), copied verbatim in the file "COPYING".
6+
*
7+
* See http://alice-o2.web.cern.ch/license for full licensing information.
8+
*
9+
* In applying this license CERN does not waive the privileges and immunities
10+
* granted to it by virtue of its status as an Intergovernmental Organization
11+
* or submit itself to any jurisdiction.
12+
*/
13+
14+
const Sequelize = require('sequelize');
15+
16+
module.exports = (sequelize) => {
17+
const DataPassVersion = sequelize.define(
18+
'DataPassVersion',
19+
{
20+
id: {
21+
type: Sequelize.INTEGER,
22+
allowNull: false,
23+
primaryKey: true,
24+
autoIncrement: true,
25+
},
26+
description: {
27+
type: Sequelize.TEXT,
28+
allowNull: false,
29+
unique: true,
30+
},
31+
outputSize: {
32+
type: Sequelize.BIGINT,
33+
},
34+
reconstructedEventsCount: {
35+
type: Sequelize.INTEGER,
36+
},
37+
lastSeen: {
38+
type: Sequelize.INTEGER,
39+
comment: 'Change of this property in MonALISA means recent job activity that changed data pass version properties',
40+
},
41+
},
42+
{ tableName: 'data_pass_versions' },
43+
);
44+
45+
DataPassVersion.associate = (models) => {
46+
DataPassVersion.belongsTo(models.DataPass, { as: 'dataPass' });
47+
};
48+
49+
return DataPassVersion;
50+
};

‎lib/database/models/index.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
const Attachment = require('./attachment');
1515
const DataPass = require('./dataPass.js');
1616
const DataPassQcFlag = require('./dataPassQcFlag.js');
17+
const DataPassVersion = require('./dataPassVersion.js');
1718
const Detector = require('./detector');
1819
const DplDetector = require('./dpl/dpldetector.js');
1920
const DplProcess = require('./dpl/dplprocess.js');
@@ -51,6 +52,7 @@ module.exports = (sequelize) => {
5152
Attachment: Attachment(sequelize),
5253
DataPass: DataPass(sequelize),
5354
DataPassQcFlag: DataPassQcFlag(sequelize),
55+
DataPassVersion: DataPassVersion(sequelize),
5456
Detector: Detector(sequelize),
5557
DplDetector: DplDetector(sequelize),
5658
DplProcess: DplProcess(sequelize),

‎lib/database/models/typedefs/SequelizeDataPass.js‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,6 @@
1616
*
1717
* @property {number} id
1818
* @property {string} name
19-
* @property {string} description
20-
* @property {number} outputSize
21-
* @property {number} reconstructedEventsCount
22-
* @property {number} lastRunNumber
2319
* @property {number} runsCount
2420
* @property {number} simulationPassesCount
2521
*/

0 commit comments

Comments
 (0)