Mergers: Retry with gDebug if merging fails (#15153)

...to help understand the error in QC-1340.

I get the following with newly added test case:
```
Error in <TH1Merger::DifferentAxesMerge>: Cannot merge histograms - the histograms obj1 can extend the X axis or have different limits and underflows/overflows are present in the histogram obj2.
[ERROR] Failed to merge the input object 'obj2' of type 'TH1I and the target object 'obj1' of type 'TH1I'
Info in <TH1Merger::ExamineHistogram>: Examine histogram obj1 - labels 0 - same limits 1 - axis found 1
Info in <TH1Merger::ExamineHistogram>: Examine histogram obj2 - labels 0 - same limits 0 - axis found 1
Info in <Merge>: Histogram Merge type is 2 and new axis flag is 1
Info in <DefineNewAxis>: A new X axis has been defined Nbins=20 , [0.000000,20.000000]
Info in <TH1Merger::DifferentAxesMerge>: Merging histogram obj1 into obj1
Info in <TH1Merger::DifferentAxesMerge>: Merging histogram obj2 into obj1
Error in <TH1Merger::DifferentAxesMerge>: Cannot merge histograms - the histograms obj1 can extend the X axis or have different limits and underflows/overflows are present in the histogram obj2.
[ERROR] Merging 'obj2' and 'obj1' failed again after a retry for debugging purposes. See ROOT warnings for details.
```

What bothers me is that the ROOT error is actually printed there even before gDebug is enabled, while we see no ROOT error in the case of QC-1340, even though stderr output is redirected to infologger by o2-aliecs-executor.
Perhaps whathever is triggering the error does not have an error log associated with it in ROOT?

Author: knopers8

Utilities/Mergers/src/MergerAlgorithm.cxx

@@ -95,6 +95,52 @@ auto matchCollectedToPairs(const std::vector<TObject*>& targetObjects, const std
   return matchedObjects;
 }
 
+// calls the default Merge methods of TObjects
+Long64_t mergeDefault(TObject* const target, TObject* const other)
+{
+  Long64_t errorCode = 0;
+
+  TObjArray otherCollection;
+  otherCollection.SetOwner(false);
+  otherCollection.Add(other);
+
+  if (target->InheritsFrom(TH1::Class())) {
+    // this includes TH1, TH2, TH3
+    auto targetTH1 = reinterpret_cast<TH1*>(target);
+    if (targetTH1->TestBit(TH1::kIsAverage)) {
+      // Merge() does not support averages, we have to use Add()
+      // this will break if collection.size != 1
+      if (auto otherTH1 = dynamic_cast<TH1*>(otherCollection.First())) {
+        errorCode = targetTH1->Add(otherTH1);
+      }
+    } else {
+      // Add() does not support histograms with labels, thus we resort to Merge() by default
+      errorCode = targetTH1->Merge(&otherCollection);
+    }
+  } else if (target->InheritsFrom(THnBase::Class())) {
+    // this includes THn and THnSparse
+    errorCode = reinterpret_cast<THnBase*>(target)->Merge(&otherCollection);
+  } else if (target->InheritsFrom(TTree::Class())) {
+    auto targetTree = reinterpret_cast<TTree*>(target);
+    auto otherTree = reinterpret_cast<TTree*>(other);
+    auto targetTreeSize = estimateTreeSize(targetTree);
+    auto otherTreeSize = estimateTreeSize(otherTree);
+    if (auto totalSize = targetTreeSize + otherTreeSize; totalSize > 100000000) {
+      LOG(warn) << "The tree '" << targetTree->GetName() << "' would be larger than 100MB (" << totalSize << "B) after merging, skipping to let the system survive";
+      errorCode = 0;
+    } else {
+      errorCode = targetTree->Merge(&otherCollection);
+    }
+  } else if (target->InheritsFrom(TGraph::Class())) {
+    errorCode = reinterpret_cast<TGraph*>(target)->Merge(&otherCollection);
+  } else if (target->InheritsFrom(TEfficiency::Class())) {
+    errorCode = reinterpret_cast<TEfficiency*>(target)->Merge(&otherCollection);
+  } else {
+    LOG(warn) << "Object '" + std::string(target->GetName()) + "' with type '" + std::string(target->ClassName()) + "' is not one of the mergeable types, skipping";
+  }
+  return errorCode;
+}
+
 void merge(TObject* const target, TObject* const other)
 {
   if (target == nullptr) {
@@ -158,48 +204,23 @@ void merge(TObject* const target, TObject* const other)
     }
 
   } else {
-    Long64_t errorCode = 0;
-    TObjArray otherCollection;
-    otherCollection.SetOwner(false);
-    otherCollection.Add(other);
-
-    if (target->InheritsFrom(TH1::Class())) {
-      // this includes TH1, TH2, TH3
-      auto targetTH1 = reinterpret_cast<TH1*>(target);
-      if (targetTH1->TestBit(TH1::kIsAverage)) {
-        // Merge() does not support averages, we have to use Add()
-        // this will break if collection.size != 1
-        if (auto otherTH1 = dynamic_cast<TH1*>(otherCollection.First())) {
-          errorCode = targetTH1->Add(otherTH1);
-        }
-      } else {
-        // Add() does not support histograms with labels, thus we resort to Merge() by default
-        errorCode = targetTH1->Merge(&otherCollection);
-      }
-    } else if (target->InheritsFrom(THnBase::Class())) {
-      // this includes THn and THnSparse
-      errorCode = reinterpret_cast<THnBase*>(target)->Merge(&otherCollection);
-    } else if (target->InheritsFrom(TTree::Class())) {
-      auto targetTree = reinterpret_cast<TTree*>(target);
-      auto otherTree = reinterpret_cast<TTree*>(other);
-      auto targetTreeSize = estimateTreeSize(targetTree);
-      auto otherTreeSize = estimateTreeSize(otherTree);
-      if (auto totalSize = targetTreeSize + otherTreeSize; totalSize > 100000000) {
-        LOG(warn) << "The tree '" << targetTree->GetName() << "' would be larger than 100MB (" << totalSize << "B) after merging, skipping to let the system survive";
-        errorCode = 0;
-      } else {
-        errorCode = targetTree->Merge(&otherCollection);
-      }
-    } else if (target->InheritsFrom(TGraph::Class())) {
-      errorCode = reinterpret_cast<TGraph*>(target)->Merge(&otherCollection);
-    } else if (target->InheritsFrom(TEfficiency::Class())) {
-      errorCode = reinterpret_cast<TEfficiency*>(target)->Merge(&otherCollection);
-    } else {
-      LOG(warn) << "Object '" + std::string(target->GetName()) + "' with type '" + std::string(target->ClassName()) + "' is not one of the mergeable types, skipping";
-    }
+    Long64_t errorCode = mergeDefault(target, other);
+
     if (errorCode == -1) {
       LOG(error) << "Failed to merge the input object '" + std::string(other->GetName()) + "' of type '" + std::string(other->ClassName()) //
                       + " and the target object '" + std::string(target->GetName()) + "' of type '" + std::string(target->ClassName()) + "'";
+
+      // we retry with debug options enabled in ROOT in hopes to get some logs explaining the issue
+      gDebug = true;
+      errorCode = mergeDefault(target, other);
+      gDebug = false;
+      if (errorCode == -1) {
+        LOG(error) << "Merging '" + std::string(other->GetName()) + "' and '" + std::string(target->GetName()) //
+                        + "' failed again after a retry for debugging purposes. See ROOT warnings for details.";
+      } else {
+        LOG(warn) << "Merging '" + std::string(other->GetName()) + "' and '" + std::string(target->GetName()) //
+                       + "' succeeded after retrying for debugging purposes.";
+      }
     }
   }
 }

Utilities/Mergers/test/test_Algorithm.cxx

@@ -97,6 +97,22 @@ BOOST_AUTO_TEST_CASE(MergerSingularObjects)
     delete other;
     delete target;
   }
+  {
+    // mismatching axes - merging should fail.
+    // we should run again merging with gDebug enabled to see more logs from ROOT (tested only manually by visual inspection of logs)
+    TH1I* target = new TH1I("obj1", "obj1", bins, min, max);
+    target->Fill(5);
+    TH1I* other = new TH1I("obj2", "obj2", bins, max, max + 10);
+    other->Fill(2);
+    other->Fill(2);
+
+    BOOST_CHECK_NO_THROW(algorithm::merge(target, other));
+    BOOST_CHECK_EQUAL(target->GetBinContent(target->FindBin(2)), 0);
+    BOOST_CHECK_EQUAL(target->GetBinContent(target->FindBin(5)), 1);
+
+    delete other;
+    delete target;
+  }
   {
     TH2I* target = new TH2I("obj1", "obj1", bins, min, max, bins, min, max);
     target->Fill(5, 5);