Opt.to detect TF from orbit and SOX info instead of TType

Author: shahor02

Detectors/Raw/README.md

@@ -318,7 +318,8 @@ o2-raw-file-reader-workflow
   --part-per-hbf                        FMQ parts per superpage (default) of HBF
   --raw-channel-config arg              optional raw FMQ channel for non-DPL output
   --cache-data                          cache data at 1st reading, may require excessive memory!!!
-  --detect-tf0                          autodetect HBFUtils start Orbit/BC from 1st TF seen
+  --detect-tf0                          autodetect HBFUtils start Orbit/BC from 1st TF seen (at SOX)
+  --calculate-tf-start                  calculate TF start from orbit instead of using TType
   --configKeyValues arg                 semicolon separated key=value strings
 
   # to suppress various error checks / reporting
@@ -331,6 +332,9 @@ o2-raw-file-reader-workflow
   --nocheck-tf-per-link                 ignore /Number of TFs is less than expected/
   --nocheck-hbf-jump                    ignore /Wrong HBF orbit increment/
   --nocheck-no-spage-for-tf             ignore /TF does not start by new superpage/
+  --nocheck-no-sox                      ignore /No SOX found on 1st page/
+  --nocheck-tf-start-mismatch           ignore /Mismatch between TType-flagged and calculated new TF start/
+
 ```
 
 The workflow takes an input from the configuration file (as described in `RawFileReader` section), reads the data and sends them as DPL messages
@@ -369,6 +373,7 @@ Options:
   -v [ --verbosity ] arg (=0)    1: long report, 2 or 3: print or dump all RDH
   -s [ --spsize ]    arg (=1048576) nominal super-page size in bytes
   --detect-tf0                      autodetect HBFUtils start Orbit/BC from 1st TF seen
+  --calculate-tf-start              calculate TF start from orbit instead of using TType
   --rorc                            impose RORC as default detector mode
   --configKeyValues arg             semicolon separated key=value strings
   --nocheck-packet-increment        ignore /Wrong RDH.packetCounter increment/
@@ -380,6 +385,8 @@ Options:
   --nocheck-tf-per-link             ignore /Number of TFs is less than expected/
   --nocheck-hbf-jump                ignore /Wrong HBF orbit increment/
   --nocheck-no-spage-for-tf         ignore /TF does not start by new superpage/
+  --nocheck-no-sox                  ignore /No SOX found on 1st page/
+  --nocheck-tf-start-mismatch       ignore /Mismatch between TType-flagged and calculated new TF start/
   (input files are optional if config file was provided)
 ```
 

Detectors/Raw/include/DetectorsRaw/RawFileReader.h

@@ -34,6 +34,22 @@ namespace raw
 
 using IR = o2::InteractionRecord;
 
+struct ReaderInp {
+  std::string inifile{};
+  std::string rawChannelConfig{};
+  size_t spSize = 1024L * 1024L;
+  size_t bufferSize = 1024L * 1024L;
+  int loop = 1;
+  uint32_t delay_us = 0;
+  uint32_t errMap = 0xffffffff;
+  uint32_t minTF = 0;
+  uint32_t maxTF = 0xffffffff;
+  bool partPerSP = true;
+  bool cache = false;
+  bool autodetectTF0 = false;
+  bool preferCalcTF = false;
+};
+
 class RawFileReader
 {
   using LinkSpec_t = uint64_t; // = (origin<<32) | LinkSubSpec
@@ -52,6 +68,8 @@ class RawFileReader
                   ErrWrongNumberOfTF,
                   ErrHBFJump,
                   ErrNoSuperPageForTF,
+                  ErrNoSOX,
+                  ErrMismatchTF,
                   NErrorsDefined
   };
 
@@ -61,15 +79,17 @@ class RawFileReader
 
   static constexpr std::string_view ErrNames[] = {
     // long names for error codes
-    "Wrong RDH.packetCounter increment",     // ErrWrongPacketCounterIncrement
-    "Wrong RDH.pageCnt increment",           // ErrWrongPageCounterIncrement
-    "RDH.stop set of 1st HBF page",          // ErrHBFStopOnFirstPage
-    "New HBF starts w/o closing old one",    // ErrHBFNoStop
-    "Data does not start with TF/HBF",       // ErrWrongFirstPage
-    "Number of HBFs per TF not as expected", // ErrWrongHBFsPerTF
-    "Number of TFs is less than expected",   // ErrWrongNumberOfTF
-    "Wrong HBF orbit increment",             // ErrHBFJump
-    "TF does not start by new superpage"     // ErrNoSuperPageForTF
+    "Wrong RDH.packetCounter increment",                   // ErrWrongPacketCounterIncrement
+    "Wrong RDH.pageCnt increment",                         // ErrWrongPageCounterIncrement
+    "RDH.stop set of 1st HBF page",                        // ErrHBFStopOnFirstPage
+    "New HBF starts w/o closing old one",                  // ErrHBFNoStop
+    "Data does not start with TF/HBF",                     // ErrWrongFirstPage
+    "Number of HBFs per TF not as expected",               // ErrWrongHBFsPerTF
+    "Number of TFs is less than expected",                 // ErrWrongNumberOfTF
+    "Wrong HBF orbit increment",                           // ErrHBFJump
+    "TF does not start by new superpage",                  // ErrNoSuperPageForTF
+    "No SOX found on 1st page",                            // ErrNoSOX
+    "Mismatch between flagged and calculated new TF start" // ErrMismatchTF
   };
   static constexpr std::string_view ErrNamesShort[] = {
     // short names for error codes
@@ -81,7 +101,9 @@ class RawFileReader
     "hbf-per-tf",       // ErrWrongHBFsPerTF
     "tf-per-link",      // ErrWrongNumberOfTF
     "hbf-jump",         // ErrHBFJump
-    "no-spage-for-tf"   // ErrNoSuperPageForTF
+    "no-spage-for-tf",  // ErrNoSuperPageForTF
+    "no-sox",           // ErrNoSOX
+    "tf-start-mismatch" // ErrMismatchTF
   };
   static constexpr bool ErrCheckDefaults[] = {
     true,  // ErrWrongPacketCounterIncrement
@@ -92,7 +114,9 @@ class RawFileReader
     true,  // ErrWrongHBFsPerTF
     true,  // ErrWrongNumberOfTF
     true,  // ErrHBFJump
-    false  // ErrNoSuperPageForTF
+    false, // ErrNoSuperPageForTF
+    true,  // ErrNoSOX
+    true,  // ErrMismatchTF
   };
   //================================================================================
 
@@ -115,12 +139,12 @@ class RawFileReader
            StartHB = 0x1 << 1,
            StartSP = 0x1 << 2,
            EndHB = 0x1 << 3 };
-    size_t offset = 0;   //! where data of the block starts
-    uint32_t size = 0;   //! block size
-    uint32_t tfID = 0;   //! tf counter (from 0)
-    IR ir = 0;           //! ir starting the block
-    uint16_t fileID = 0; //! file id where the block is located
-    uint8_t flags = 0;   //! different flags
+    size_t offset = 0;                 //! where data of the block starts
+    uint32_t size = 0;                 //! block size
+    uint32_t tfID = 0;                 //! tf counter (from 0)
+    IR ir = 0;                         //! ir starting the block
+    uint16_t fileID = 0;               //! file id where the block is located
+    uint8_t flags = 0;                 //! different flags
     std::unique_ptr<char[]> dataCache; //! optional cache for fast access
     LinkBlock() = default;
     LinkBlock(int fid, size_t offs) : offset(offs), fileID(fid) {}
@@ -138,7 +162,8 @@ class RawFileReader
 
   //=====================================================================================
   struct LinkData {
-    RDHAny rdhl;               //! RDH with the running info of the last RDH seen
+    RDHAny rdhl; //! RDH with the running info of the last RDH seen
+    o2::InteractionRecord irOfSOX{};
     LinkSpec_t spec = 0;       //! Link subspec augmented by its origin
     LinkSubSpec_t subspec = 0; //! subspec according to DataDistribution
     uint32_t nTimeFrames = 0;  //!
@@ -244,8 +269,11 @@ class RawFileReader
 
   void imposeFirstTF(uint32_t orbit, uint16_t bc);
   void setTFAutodetect(FirstTFDetection v) { mFirstTFAutodetect = v; }
+  void setPreferCalculatedTFStart(bool v) { mPreferCalculatedTFStart = v; }
   FirstTFDetection getTFAutodetect() const { return mFirstTFAutodetect; }
 
+  void setIROfSOX(const o2::InteractionRecord& ir);
+
   static o2::header::DataOrigin getDataOrigin(const std::string& ors);
   static o2::header::DataDescription getDataDescription(const std::string& ors);
   static InputsMap parseInput(const std::string& confUri);
@@ -260,38 +288,36 @@ class RawFileReader
   static constexpr o2::header::DataOrigin DEFDataOrigin = o2::header::gDataOriginFLP;
   static constexpr o2::header::DataDescription DEFDataDescription = o2::header::gDataDescriptionRawData;
   static constexpr ReadoutCardType DEFCardType = CRU;
-
   o2::header::DataOrigin mDefDataOrigin = DEFDataOrigin;                //!
   o2::header::DataDescription mDefDataDescription = DEFDataDescription; //!
   ReadoutCardType mDefCardType = CRU;                                   //!
-
-  std::vector<std::string> mFileNames;               //! input file names
-  std::vector<FILE*> mFiles;                         //! input file handlers
-  std::vector<std::unique_ptr<char[]>> mFileBuffers; //! buffers for input files
-  std::vector<OrigDescCard> mDataSpecs;              //! data origin and description for every input file + readout card type
+  std::vector<std::string> mFileNames;                                  //! input file names
+  std::vector<FILE*> mFiles;                                            //! input file handlers
+  std::vector<std::unique_ptr<char[]>> mFileBuffers;                    //! buffers for input files
+  std::vector<OrigDescCard> mDataSpecs;                                 //! data origin and description for every input file + readout card type
   bool mInitDone = false;
   bool mEmpty = true;
-  std::unordered_map<LinkSpec_t, int> mLinkEntries; //! mapping between RDH specs and link entry in the mLinksData
-  std::vector<LinkData> mLinksData;                 //! info on links data in the files
-  std::vector<int> mOrderedIDs;                     //! links entries ordered in Specs
-  uint32_t mMaxTFToRead = 0xffffffff;               //! max TFs to process
-  uint32_t mNTimeFrames = 0;                        //! total number of time frames
-  uint32_t mNextTF2Read = 0;                        //! next TF to read
-  uint32_t mOrbitMin = 0xffffffff;                  //! lowest orbit seen by any link
-  uint32_t mOrbitMax = 0;                           //! highest orbit seen by any link
-  size_t mBufferSize = 5 * 1024UL;                  //! size of the buffer for files reading
-  int mNominalSPageSize = 0x1 << 20;                //! expected super-page size in B
-  int mCurrentFileID = 0;                           //! current file being processed
-  long int mPosInFile = 0;                          //! current position in the file
-  bool mMultiLinkFile = false;                      //! was > than 1 link seen in the file?
-  bool mCacheData = false;                          //! cache data to block after 1st scan (may require excessive memory, use with care)
-  uint32_t mCheckErrors = 0;                        //! mask for errors to check
+  std::unordered_map<LinkSpec_t, int> mLinkEntries;                 //! mapping between RDH specs and link entry in the mLinksData
+  std::vector<LinkData> mLinksData;                                 //! info on links data in the files
+  std::vector<int> mOrderedIDs;                                     //! links entries ordered in Specs
+  uint32_t mMaxTFToRead = 0xffffffff;                               //! max TFs to process
+  uint32_t mNTimeFrames = 0;                                        //! total number of time frames
+  uint32_t mNextTF2Read = 0;                                        //! next TF to read
+  uint32_t mOrbitMin = 0xffffffff;                                  //! lowest orbit seen by any link
+  uint32_t mOrbitMax = 0;                                           //! highest orbit seen by any link
+  size_t mBufferSize = 5 * 1024UL;                                  //! size of the buffer for files reading
+  int mNominalSPageSize = 0x1 << 20;                                //! expected super-page size in B
+  int mCurrentFileID = 0;                                           //! current file being processed
+  long int mPosInFile = 0;                                          //! current position in the file
+  bool mMultiLinkFile = false;                                      //! was > than 1 link seen in the file?
+  bool mCacheData = false;                                          //! cache data to block after 1st scan (may require excessive memory, use with care)
+  uint32_t mCheckErrors = 0;                                        //! mask for errors to check
   FirstTFDetection mFirstTFAutodetect = FirstTFDetection::Disabled; //!
+  bool mPreferCalculatedTFStart = false;                            //! prefer TFstart calculated via HBFUtils
   int mVerbosity = 0;                                               //!
   ClassDefNV(RawFileReader, 1);
 };
 
-
 } // namespace raw
 } // namespace o2
 

Detectors/Raw/src/RawFileReader.cxx

@@ -354,24 +354,50 @@ bool RawFileReader::LinkData::preprocessCRUPage(const RDHAny& rdh, bool newSPage
     ok = false;
     nErrors++;
   }
-
+  auto ir = RDHUtils::getTriggerIR(rdh);
   auto pageCnt = RDHUtils::getPageCounter(rdh);
+
   if (pageCnt == 0) {
-    if (cruDetector) {
-      auto triggerType = RDHUtils::getTriggerType(rdh);
-      newTF = (triggerType & o2::trigger::TF);
-      newHB = (triggerType & (o2::trigger::ORBIT | o2::trigger::HB)) == (o2::trigger::ORBIT | o2::trigger::HB);
+    auto triggerType = RDHUtils::getTriggerType(rdh);
+    if (!nCRUPages) { // 1st page, expect SOX
       if (triggerType & o2::trigger::SOC) {
         continuousRO = true;
+        irOfSOX = ir;
       } else if (triggerType & o2::trigger::SOT) {
         continuousRO = false;
+        irOfSOX = ir;
+      } else {
+        if (reader->mCheckErrors & (0x1 << ErrNoSOX)) {
+          LOG(ERROR) << ErrNames[ErrNoSOX];
+          ok = false;
+          nErrors++;
+        }
+      }
+      if (!irOfSOX.isDummy() && reader->getTFAutodetect() == FirstTFDetection::Pending) {
+        reader->imposeFirstTF(irOfSOX.orbit, irOfSOX.bc);
+      }
+    }
+    auto newTFCalc = blocks.empty() || HBU.getTF(blocks.back().ir) < HBU.getTF(ir);
+    if (cruDetector) {
+      newTF = (triggerType & o2::trigger::TF);
+      newHB = (triggerType & (o2::trigger::ORBIT | o2::trigger::HB)) == (o2::trigger::ORBIT | o2::trigger::HB);
+      if (newTFCalc != newTF && (reader->mCheckErrors & (0x1 << ErrMismatchTF))) {
+        LOG(ERROR) << ErrNames[ErrMismatchTF];
+        ok = false;
+        nErrors++;
+      }
+      if (reader->mPreferCalculatedTFStart) {
+        newTF = newTFCalc;
+        if (newTF) {
+          newHB = true;
+        }
       }
     } else {
       newHB = true; // in RORC detectors treat each trigger as a HBF
-      if (blocks.empty() || HBU.getTF(blocks.back().ir) < HBU.getTF(RDHUtils::getTriggerIR(rdh))) {
+      if (newTFCalc) {
         newTF = true;
+        // continuousRO = false;
       }
-      continuousRO = false;
     }
   } else if (reader->mCheckErrors & (0x1 << ErrWrongPageCounterIncrement)) {
     // check increasing pageCnt
@@ -687,7 +713,7 @@ bool RawFileReader::init()
     }
   }
   std::sort(mOrderedIDs.begin(), mOrderedIDs.end(),
-            [& links = mLinksData](int a, int b) { return links[a].spec < links[b].spec; });
+            [&links = mLinksData](int a, int b) { return links[a].spec < links[b].spec; });
 
   size_t maxSP = 0, maxTF = 0;
 

Detectors/Raw/src/RawFileReaderWorkflow.cxx

@@ -45,18 +45,16 @@ namespace o2h = o2::header;
 class RawReaderSpecs : public o2f::Task
 {
  public:
-  explicit RawReaderSpecs(const std::string& config, int loop = 1, uint32_t delay_us = 0,
-                          uint32_t errmap = 0xffffffff, uint32_t minTF = 0, uint32_t maxTF = 0xffffffff, bool partPerSP = true, bool cache = false, bool autodetectTF0 = false,
-                          size_t spSize = 1024L * 1024L, size_t buffSize = 5 * 1024UL,
-                          const std::string& rawChannelName = "")
-    : mLoop(loop < 0 ? INT_MAX : (loop < 1 ? 1 : loop)), mDelayUSec(delay_us), mMinTFID(minTF), mMaxTFID(maxTF), mPartPerSP(partPerSP), mReader(std::make_unique<o2::raw::RawFileReader>(config, 0, buffSize)), mRawChannelName(rawChannelName)
+  explicit RawReaderSpecs(const ReaderInp& rinp)
+    : mLoop(rinp.loop < 0 ? INT_MAX : (rinp.loop < 1 ? 1 : rinp.loop)), mDelayUSec(rinp.delay_us), mMinTFID(rinp.minTF), mMaxTFID(rinp.maxTF), mPartPerSP(rinp.partPerSP), mReader(std::make_unique<o2::raw::RawFileReader>(rinp.inifile, 0, rinp.bufferSize)), mRawChannelName(rinp.rawChannelConfig)
   {
-    mReader->setCheckErrors(errmap);
-    mReader->setMaxTFToRead(maxTF);
-    mReader->setNominalSPageSize(spSize);
-    mReader->setCacheData(cache);
-    mReader->setTFAutodetect(autodetectTF0 ? RawFileReader::FirstTFDetection::Pending : RawFileReader::FirstTFDetection::Disabled);
-    LOG(INFO) << "Will preprocess files with buffer size of " << buffSize << " bytes";
+    mReader->setCheckErrors(rinp.errMap);
+    mReader->setMaxTFToRead(rinp.maxTF);
+    mReader->setNominalSPageSize(rinp.spSize);
+    mReader->setCacheData(rinp.cache);
+    mReader->setTFAutodetect(rinp.autodetectTF0 ? RawFileReader::FirstTFDetection::Pending : RawFileReader::FirstTFDetection::Disabled);
+    mReader->setPreferCalculatedTFStart(rinp.preferCalcTF);
+    LOG(INFO) << "Will preprocess files with buffer size of " << rinp.bufferSize << " bytes";
     LOG(INFO) << "Number of loops over whole data requested: " << mLoop;
     for (int i = NTimers; i--;) {
       mTimer[i].Stop();
@@ -245,16 +243,15 @@ class RawReaderSpecs : public o2f::Task
   TStopwatch mTimer[NTimers];
 };
 
-o2f::DataProcessorSpec getReaderSpec(std::string config, int loop, uint32_t delay_us, uint32_t errmap,
-                                     uint32_t minTF, uint32_t maxTF, bool partPerSP, bool cache, bool autodetectTF0, size_t spSize, size_t buffSize, const std::string& rawChannelConfig)
+o2f::DataProcessorSpec getReaderSpec(const ReaderInp& rinp)
 {
   // check which inputs are present in files to read
   o2f::DataProcessorSpec spec;
   spec.name = "raw-file-reader";
   std::string rawChannelName = "";
-  if (rawChannelConfig.empty()) {
-    if (!config.empty()) {
-      auto conf = o2::raw::RawFileReader::parseInput(config);
+  if (rinp.rawChannelConfig.empty()) {
+    if (!rinp.inifile.empty()) {
+      auto conf = o2::raw::RawFileReader::parseInput(rinp.inifile);
       for (const auto& entry : conf) {
         const auto& ordescard = entry.first;
         if (!entry.second.empty()) { // origin and decription for files to process
@@ -263,29 +260,28 @@ o2f::DataProcessorSpec getReaderSpec(std::string config, int loop, uint32_t dela
       }
     }
   } else {
-    auto nameStart = rawChannelConfig.find("name=");
+    auto nameStart = rinp.rawChannelConfig.find("name=");
     if (nameStart == std::string::npos) {
       throw std::runtime_error("raw channel name is not provided");
     }
     nameStart += strlen("name=");
-    auto nameEnd = rawChannelConfig.find(",", nameStart + 1);
+    auto nameEnd = rinp.rawChannelConfig.find(",", nameStart + 1);
     if (nameEnd == std::string::npos) {
-      nameEnd = rawChannelConfig.size();
+      nameEnd = rinp.rawChannelConfig.size();
     }
-    rawChannelName = rawChannelConfig.substr(nameStart, nameEnd - nameStart);
-    spec.options = {o2f::ConfigParamSpec{"channel-config", o2f::VariantType::String, rawChannelConfig, {"Out-of-band channel config"}}};
-    LOG(INFO) << "Will send output to non-DPL channel " << rawChannelConfig;
+    rawChannelName = rinp.rawChannelConfig.substr(nameStart, nameEnd - nameStart);
+    spec.options = {o2f::ConfigParamSpec{"channel-config", o2f::VariantType::String, rinp.rawChannelConfig, {"Out-of-band channel config"}}};
+    LOG(INFO) << "Will send output to non-DPL channel " << rinp.rawChannelConfig;
   }
 
-  spec.algorithm = o2f::adaptFromTask<RawReaderSpecs>(config, loop, delay_us, errmap, minTF, maxTF, partPerSP, cache, autodetectTF0, spSize, buffSize, rawChannelName);
+  spec.algorithm = o2f::adaptFromTask<RawReaderSpecs>(rinp);
 
   return spec;
 }
 
-o2f::WorkflowSpec o2::raw::getRawFileReaderWorkflow(std::string inifile, int loop, uint32_t delay_us, uint32_t errmap, uint32_t minTF, uint32_t maxTF,
-                                                    bool partPerSP, bool cache, bool autodetectTF0, size_t spSize, size_t buffSize, const std::string& rawChannelConfig)
+o2f::WorkflowSpec o2::raw::getRawFileReaderWorkflow(const ReaderInp& rinp)
 {
   o2f::WorkflowSpec specs;
-  specs.emplace_back(getReaderSpec(inifile, loop, delay_us, errmap, minTF, maxTF, partPerSP, cache, autodetectTF0, spSize, buffSize, rawChannelConfig));
+  specs.emplace_back(getReaderSpec(rinp));
   return specs;
 }