Add support for the publication of the list of all published objects.

This is needed by the CCDB backend as we have no ways of asking for the
list of objects. It introduces the (first) so-called system object, i.e.
an object that is transmitted by tasks behind the scene the same way
the explicit MonitorObjects are.
At the moment, the list of objects is a comma separated list of names.
It is interpreted by the CCDB backend to provide the list of published
objects.
(QC-53)

Author: Barthelemy

Framework/include/QualityControl/MonitorObject.h

@@ -140,6 +140,10 @@ class MonitorObject : public TObject
       return clone;
     }
 
+  // Names of special objects published by the framework for each task, behind the scene.
+  public:
+    static constexpr char SYSTEM_OBJECT_PUBLICATION_LIST[] = "objectsList"; // list of objects published by the task
+
   private:
     std::string mName;
     Quality mQuality;

Framework/include/QualityControl/ObjectsManager.h

@@ -11,6 +11,7 @@
 #include "QualityControl/Quality.h"
 #include "QualityControl/MonitorObject.h"
 #include "QualityControl/TaskConfig.h"
+#include <TObjString.h>
 
 namespace o2 {
 namespace quality_control {
@@ -27,9 +28,6 @@ class ObjectsManager
     friend class TaskControl; // TaskControl must be able to call "publish()" whenever needed. Nobody else can.
 
   public:
-    /// Default constructor
-    ObjectsManager();
-
     ObjectsManager(TaskConfig &taskConfig);
 
     /// Destructor
@@ -38,6 +36,7 @@ class ObjectsManager
     void startPublishing(TObject *obj, std::string objectName = "");
 
     void setQuality(std::string objectName, Quality quality);
+    // todo stop publishing
 
     Quality getQuality(std::string objectName);
 
@@ -72,9 +71,15 @@ class ObjectsManager
     iterator end()
     { return mMonitorObjects.end(); }
 
+  private:
+    void UpdateIndex(const std::string &nonEmptyName) ;
+
   private:
     std::map<std::string /*object name*/, quality_control::core::MonitorObject * /* object */> mMonitorObjects;
     std::string mTaskName;
+    // todo make it a vector of string when support added
+    TObjString mObjectsList; // the list of objects we publish. (comma separated)
+                       // Possibly needed on the client side to know what was there at a given time.
 };
 
 } // namespace core

Framework/src/CcdbDatabase.cxx

@@ -3,7 +3,11 @@
 //
 
 #include <TMessage.h>
+#include <TObjString.h>
 #include "QualityControl/CcdbDatabase.h"
+#include <regex>
+#include <boost/algorithm/string/split.hpp>
+#include <boost/algorithm/string.hpp>
 
 namespace o2 {
 namespace quality_control {
@@ -253,13 +257,27 @@ void CcdbDatabase::prepareTaskDataContainer(std::string taskName)
 
 std::vector<std::string> CcdbDatabase::getListOfTasksWithPublications()
 {
-  std::vector<string> result{"daqTask"};
+  std::vector<string> result{"daqTask"}; // TODO we need the "ls" feature in CCDB to list the top level nodes.
   return result;
 }
 
 std::vector<std::string> CcdbDatabase::getPublishedObjectNames(std::string taskName)
 {
-  std::vector<string> result{"IDs", "numberSubBlocks", "payloadSize", "PayloadSizeSubBlocks"};
+  std::vector<string> result;
+  // we use the "index" string to know what objects are published.
+  // get the information from the CCDB itself.
+  core::MonitorObject *mo = retrieve(taskName, core::MonitorObject::SYSTEM_OBJECT_PUBLICATION_LIST);
+  auto *indexString = dynamic_cast<TObjString *>(mo->getObject());
+  if (indexString) {
+    string s = indexString->GetString().Data();
+    boost::algorithm::split(result, s, boost::algorithm::is_any_of(","),boost::algorithm::token_compress_on);
+    // sanitize : remove system objects object and empty objects
+    result.erase(std::remove(result.begin(), result.end(), ""));
+    result.erase(std::remove(result.begin(), result.end(), core::MonitorObject::SYSTEM_OBJECT_PUBLICATION_LIST));
+  } else {
+    cerr << "ok we should do something here. The 'index' object must be a TObjString" << endl;
+  }
+
   return result;
 }
 

Framework/src/MonitorObject.cxx

@@ -11,6 +11,8 @@ namespace o2 {
 namespace quality_control {
 namespace core {
 
+    constexpr char MonitorObject::SYSTEM_OBJECT_PUBLICATION_LIST[];
+
 MonitorObject::MonitorObject()
   : TObject(), mName(""), mQuality(Quality::Null), mObject(nullptr), mTaskName(""), mIsOwner(true)
 {

Framework/src/MySqlDatabase.cxx

@@ -25,7 +25,7 @@ namespace quality_control {
 namespace repository {
 
 MySqlDatabase::MySqlDatabase()
-    : mServer(nullptr), queueSize(0)
+  : mServer(nullptr), queueSize(0)
 {
   lastStorage.reset();
 }
@@ -51,8 +51,8 @@ void MySqlDatabase::connect(std::string host, std::string database, std::string
   mServer = new TMySQLServer(connectionString.str().c_str(), username.c_str(), password.c_str());
   if (!mServer || mServer->GetErrorCode()) {
     string s = "Failed to connect to the database\n";
-    if(mServer->GetErrorCode()) {
-      s+= mServer->GetErrorMsg();
+    if (mServer->GetErrorCode()) {
+      s += mServer->GetErrorMsg();
     }
     BOOST_THROW_EXCEPTION(FatalException() << errinfo_details(s));
   } else {
@@ -65,31 +65,32 @@ void MySqlDatabase::prepareTaskDataContainer(std::string taskName)
   // one object per run
   string query;
   query += "CREATE TABLE IF NOT EXISTS `data_" + taskName
-      + "` (object_name CHAR(64), updatetime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, data LONGBLOB, size INT, run INT, "
-          "fill INT, PRIMARY KEY(object_name, run)) ENGINE=MyISAM";
+           +
+           "` (object_name CHAR(64), updatetime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, data LONGBLOB, size INT, run INT, "
+             "fill INT, PRIMARY KEY(object_name, run)) ENGINE=MyISAM";
   if (!execute(query)) {
     BOOST_THROW_EXCEPTION(FatalException() << errinfo_details("Failed to create data table"));
   } else {
     QcInfoLogger::GetInstance() << "Create data table for task " << taskName << infologger::endm;
   }
 }
 
-void MySqlDatabase::store(MonitorObject* mo)
+void MySqlDatabase::store(MonitorObject *mo)
 {
   // TODO we take ownership here to delete later -> clearly to be improved
   // we execute grouped insertions. Here we just register that we should keep this mo in memory.
   mObjectsQueue[mo->getTaskName()].push_back(mo);
   queueSize++;
-  if(queueSize > 100 || lastStorage.getTime() > 10 /*sec*/) { // TODO use a configuration to set the max limits
+  if (queueSize > 1 || lastStorage.getTime() > 10 /*sec*/) { // TODO use a configuration to set the max limits
     storeQueue();
   }
 }
 
 void MySqlDatabase::storeQueue()
 {
-  cout << "Database queue will now be processed ("<< queueSize << " objects)" << endl;
+  cout << "Database queue will now be processed (" << queueSize << " objects)" << endl;
 
-  for (auto& kv : mObjectsQueue) {
+  for (auto &kv : mObjectsQueue) {
     storeForTask(kv.first);
   }
   mObjectsQueue.clear();
@@ -99,9 +100,9 @@ void MySqlDatabase::storeQueue()
 
 void MySqlDatabase::storeForTask(std::string taskName)
 {
-  vector<MonitorObject*> objects = mObjectsQueue[taskName];
+  vector<MonitorObject *> objects = mObjectsQueue[taskName];
 
-  if(objects.size() == 0) {
+  if (objects.size() == 0) {
     return;
   }
 
@@ -111,13 +112,7 @@ void MySqlDatabase::storeForTask(std::string taskName)
   // build statement string
   string query;
   query += "REPlACE INTO `data_" + taskName
-           + "` (object_name, data, size, run, fill) values ";
-  for(size_t i = 0 ; i < objects.size() ; i++) {
-    if(i != 0) {
-      query += ",";
-    }
-    query += "(?,?,octet_length(data),?,?)";
-  }
+           + "` (object_name, data, size, run, fill) values (?,?,octet_length(data),?,?)";
 
   // Prepare statement
   TMySQLStatement *statement;
@@ -135,67 +130,68 @@ void MySqlDatabase::storeForTask(std::string taskName)
   }
 
   // Assign data
-  int i=0;
+//  int i = 0;
   TMessage message(kMESS_OBJECT);
-  for(auto mo : objects) {
+  for (auto mo : objects) {
     message.Reset();
     message.WriteObjectAny(mo, mo->IsA());
     statement->NextIteration();
-    statement->SetString(i+0, mo->getName().c_str());
-    statement->SetBinary(i+1, message.Buffer(), message.Length(), message.Length());
-    statement->SetInt(i+2, 0);
-    statement->SetInt(i+3, 0);
+    statement->SetString(0, mo->getName().c_str());
+    statement->SetBinary(1, message.Buffer(), message.Length(), message.Length());
+    statement->SetInt(2, 0);
+    statement->SetInt(3, 0);
   }
   statement->Process();
   delete statement;
 
-  for(auto mo : objects) {
+  for (auto mo : objects) {
     delete mo;
   }
   objects.clear();
 }
 
-o2::quality_control::core::MonitorObject* MySqlDatabase::retrieve(std::string taskName, std::string objectName)
+o2::quality_control::core::MonitorObject *MySqlDatabase::retrieve(std::string taskName, std::string objectName)
 {
   string query;
   TMySQLStatement *statement = nullptr;
 
   query += "SELECT object_name, data, updatetime, run, fill FROM data_" + taskName + " WHERE object_name = ?";
-  statement = (TMySQLStatement*) mServer->Statement(query.c_str());
+  statement = (TMySQLStatement *) mServer->Statement(query.c_str());
   if (mServer->IsError()) {
-    if (statement)
+    if (statement) {
       delete statement;
+    }
     BOOST_THROW_EXCEPTION(
-        DatabaseException() << errinfo_details("Encountered an error when creating statement in MySqlDatabase")
-            << errinfo_db_message(mServer->GetErrorMsg()) << errinfo_db_errno(mServer->GetErrorCode()));
+      DatabaseException() << errinfo_details("Encountered an error when creating statement in MySqlDatabase")
+                          << errinfo_db_message(mServer->GetErrorMsg()) << errinfo_db_errno(mServer->GetErrorCode()));
   }
   statement->NextIteration();
   statement->SetString(0, objectName.c_str());
 
   if (!(statement->Process() && statement->StoreResult())) {
     delete statement;
     BOOST_THROW_EXCEPTION(
-        DatabaseException()
-            << errinfo_details("Encountered an error when processing and storing results in MySqlDatabase")
-            << errinfo_db_message(mServer->GetErrorMsg()) << errinfo_db_errno(mServer->GetErrorCode()));
+      DatabaseException()
+        << errinfo_details("Encountered an error when processing and storing results in MySqlDatabase")
+        << errinfo_db_message(mServer->GetErrorMsg()) << errinfo_db_errno(mServer->GetErrorCode()));
   }
 
-  o2::quality_control::core::MonitorObject* mo = nullptr;
+  o2::quality_control::core::MonitorObject *mo = nullptr;
   if (statement->NextResultRow()) { // Consider only the first result
     string name = statement->GetString(0);
-    void* blob = nullptr;
+    void *blob = nullptr;
     Long_t blobSize;
     statement->GetBinary(1, blob, blobSize); // retrieve the data
     TDatime updatetime(statement->GetYear(2), statement->GetMonth(2), statement->GetDay(2), statement->GetHour(2),
-        statement->GetMinute(2), statement->GetSecond(2));
+                       statement->GetMinute(2), statement->GetSecond(2));
     int run = statement->IsNull(3) ? -1 : statement->GetInt(3);
     int fill = statement->IsNull(4) ? -1 : statement->GetInt(4);
 
     TMessage mess(kMESS_OBJECT);
     mess.SetBuffer(blob, blobSize, kFALSE);
     mess.SetReadMode();
     mess.Reset();
-    mo = (o2::quality_control::core::MonitorObject*) (mess.ReadObjectAny(mess.GetClass()));
+    mo = (o2::quality_control::core::MonitorObject *) (mess.ReadObjectAny(mess.GetClass()));
   }
   delete statement;
 
@@ -215,19 +211,20 @@ void MySqlDatabase::disconnect()
   }
 }
 
-TMySQLResult* MySqlDatabase::query(string sql)
+TMySQLResult *MySqlDatabase::query(string sql)
 {
   if (mServer) {
-    return ((TMySQLResult*) mServer->Query(sql.c_str()));
+    return ((TMySQLResult *) mServer->Query(sql.c_str()));
   } else {
     return (nullptr);
   }
 }
 
 bool MySqlDatabase::execute(string sql)
 {
-  if (mServer)
+  if (mServer) {
     return mServer->Exec(sql.c_str());
+  }
 
   return false;
 }
@@ -236,7 +233,7 @@ void MySqlDatabase::addIndex(string table, string column)
 {
   std::ostringstream stringStream;
   stringStream << "CREATE INDEX " << table << "_i_" << column << " on " << table << " (" << column << ")";
-  TMySQLResult* res = query(stringStream.str().c_str());
+  TMySQLResult *res = query(stringStream.str().c_str());
   if (res) {
     delete (res);
   } else {
@@ -249,12 +246,12 @@ std::vector<std::string> MySqlDatabase::getPublishedObjectNames(std::string task
   std::vector<std::string> result;
 
   string queryString = "select distinct object_name from ";
-  queryString += taskName;
+  queryString += "data_" + taskName;
 
-  TMySQLResult* mysqlResult = query(queryString);
+  TMySQLResult *mysqlResult = query(queryString);
   if (mysqlResult) { // only if we got a result
-    TMySQLRow* row;
-    while ((row = (TMySQLRow*) mysqlResult->Next())) {
+    TMySQLRow *row;
+    while ((row = (TMySQLRow *) mysqlResult->Next())) {
       result.push_back(row->GetField(0));
       delete row;
     }
@@ -263,16 +260,17 @@ std::vector<std::string> MySqlDatabase::getPublishedObjectNames(std::string task
 
   return result;
 }
+
 std::vector<std::string> MySqlDatabase::getListOfTasksWithPublications()
 {
   std::vector<std::string> result;
 
   string queryString = "select table_name from information_schema.tables where table_schema='quality_control'";
 
-  TMySQLResult* mysqlResult = query(queryString);
+  TMySQLResult *mysqlResult = query(queryString);
   if (mysqlResult) { // only if we got a result
-    TMySQLRow* row;
-    while ((row = (TMySQLRow*) mysqlResult->Next())) {
+    TMySQLRow *row;
+    while ((row = (TMySQLRow *) mysqlResult->Next())) {
       result.push_back(row->GetField(0));
       delete row;
     }

Framework/src/ObjectsManager.cxx

@@ -9,17 +9,15 @@
 
 using namespace o2::quality_control::core;
 using namespace AliceO2::Common;
+using namespace std;
 
 namespace o2 {
 namespace quality_control {
 namespace core {
 
-ObjectsManager::ObjectsManager() : mTaskName("anonymous task")
-{
-}
-
 ObjectsManager::ObjectsManager(TaskConfig &taskConfig) : mTaskName(taskConfig.taskName)
 {
+  startPublishing(&mObjectsList, MonitorObject::SYSTEM_OBJECT_PUBLICATION_LIST);
 }
 
 ObjectsManager::~ObjectsManager()
@@ -36,6 +34,17 @@ void ObjectsManager::startPublishing(TObject *object, std::string objectName)
   auto *newObject = new MonitorObject(nonEmptyName, object, mTaskName);
   newObject->setIsOwner(false);
   mMonitorObjects[nonEmptyName] = newObject;
+
+  //update index
+  UpdateIndex(nonEmptyName);
+}
+
+void ObjectsManager::UpdateIndex(const string &nonEmptyName)
+{
+  string newString = this->mObjectsList.GetString().Data();
+  newString += nonEmptyName;
+  newString += ",";
+  this->mObjectsList.SetString(newString.c_str());
 }
 
 void ObjectsManager::setQuality(std::string objectName, Quality quality)

Framework/src/SpyMainFrame.cxx

@@ -297,8 +297,8 @@ bool SpyMainFrame::dbIsSelected()
 void SpyMainFrame::dbRun()
 {
   // Get list of objects
-  string s = "data_";
-  s += mTaskField->GetText();
+//  string s = "data_";
+  string s = mTaskField->GetText();
   vector<string> objectNames = mDbInterface->getPublishedObjectNames(s);
 
   // UpdateList for each