Fix: Driver errors on Summit runs

Author: sandeepmittal

include/chimbuko/param/hbos_param.hpp

@@ -74,10 +74,10 @@ namespace chimbuko {
     /**
      * @brief Create new histogram locally for AD module's batch data instances
      */
-    void create_histogram(const std::vector<double>& r_times);
+    int create_histogram(const std::vector<double>& r_times);
 
 
-    void merge_histograms(const Histogram& g, const std::vector<double>& runtimes);
+    int merge_histograms(const Histogram& g, const std::vector<double>& runtimes);
 
     /**
      * @brief Combine two Histogram instances such that the resulting statistics are the union of the two

src/ad/ADOutlier.cpp

@@ -258,13 +258,16 @@ Anomalies ADOutlierHBOS::run(int step) {
     }
     if (runtimes.size() > 0) {
       if (!g.find(func_id)) { // If func_id does not exist
-        param[func_id].create_histogram(runtimes);
+        const int r = param[func_id].create_histogram(runtimes);
+        if (r < 0) {return outliers;}
       }
       else { //merge with exisiting func_id, not overwrite
 
-        param[func_id].merge_histograms(g[func_id], runtimes);
+        const int r = param[func_id].merge_histograms(g[func_id], runtimes);
+	if (r < 0) {return outliers;}
       }
     }
+    else { return outliers;}
   }
 
   //Update temp runstats to include information collected previously (synchronizes with the parameter server if connected)
@@ -332,7 +335,7 @@ unsigned long ADOutlierHBOS::compute_outliers(Anomalies &outliers,
   verboseStream << "min_score = " << min_score << std::endl;
   verboseStream << "max_score = " << max_score << std::endl;
 
-  if (out_scores_i.size() == 0) return 0;
+  if (out_scores_i.size() <= 0) {return 0;}
 
   //compute threshold
   verboseStream << "Global threshold before comparison with local threshold =  " << param[func_id].get_threshold() << std::endl;
@@ -359,28 +362,32 @@ unsigned long ADOutlierHBOS::compute_outliers(Anomalies &outliers,
       double ad_score;
 
       const int bin_ind = ADOutlierHBOS::np_digitize_get_bin_inds(runtime_i, param[func_id].bin_edges());
-      verboseStream << "bin_ind: " << bin_ind << " for runtime_i: " << runtime_i << std::endl;
+      verboseStream << "bin_ind: " << bin_ind << " for runtime_i: " << runtime_i << ", where bin_edges Size:" << param[func_id].bin_edges().size() << " & num_bins: "<< num_bins << std::endl;
       /**
        * If the sample does not belong to any bins
        * bin_ind == 0 (fall outside since it is too small)
        */
       if( bin_ind == 0){
         const double first_bin_edge = param[func_id].bin_edges().at(0);
         const double dist = first_bin_edge - runtime_i;
+	verboseStream << "First_bin_edge: " << first_bin_edge << std::endl;
         if( dist <= (bin_width * 0.05) ){
           verboseStream << runtime_i << " is on left of histogram but NOT outlier" << std::endl;
+	  if(param[func_id].counts().size() < 1) {return 0;}
           if(param[func_id].counts().at(0) == 0) { /**< Ignore zero counts */
-
+	    
             ad_score = l_threshold - 1;
             verboseStream << "corrected ad_score: " << ad_score << std::endl;
           }
           else {
             ad_score = out_scores_i.at(0);
+	    verboseStream << "ad_score: " << ad_score << std::endl;
           }
         }
         else{
           verboseStream << runtime_i << " is on left of histogram and an outlier" << std::endl;
           ad_score = max_score;
+	  verboseStream << "ad_score(max_score): " << ad_score << std::endl;
         }
 
       }
@@ -391,21 +398,23 @@ unsigned long ADOutlierHBOS::compute_outliers(Anomalies &outliers,
         const int last_idx = param[func_id].bin_edges().size() - 1;
         const double last_bin_edge = param[func_id].bin_edges().at(last_idx);
         const double dist = runtime_i - last_bin_edge;
-
+	verboseStream << "last_indx: " << last_idx << ", last_bin_edge: " << last_bin_edge << std::endl;
         if (dist <= (bin_width * 0.05)) {
-          if(param[func_id].counts().at(bin_ind) == 0) {  /**< Ignore zero counts */
+          if(param[func_id].counts().at(num_bins - 1) == 0) {   //bin_ind) == 0) {  /**< Ignore zero counts */
 
             ad_score = l_threshold - 1;
             verboseStream << "corrected ad_score: " << ad_score << std::endl;
           }
           else {
             verboseStream << runtime_i << " is on right of histogram but NOT outlier" << std::endl;
             ad_score = out_scores_i.at(num_bins - 1);
+	    verboseStream << "ad_score: " << ad_score << ", num_bins: " << num_bins << ", out_scores_i size: " << out_scores_i.size() << std::endl;
           }
         }
         else{
           verboseStream << runtime_i << " is on right of histogram and an outlier" << std::endl;
           ad_score = max_score;
+	  verboseStream << "ad_score(max_score): " << ad_score << ", num_bins: " << num_bins << ", out_scores_i size: " << out_scores_i.size() << std::endl;
         }
 
       }
@@ -419,6 +428,7 @@ unsigned long ADOutlierHBOS::compute_outliers(Anomalies &outliers,
         else {
           verboseStream << runtime_i << " maybe be an outlier" << std::endl;
           ad_score = out_scores_i.at( bin_ind - 1);
+	  verboseStream << "ad_score(else): " << ad_score << ", bin_ind: " << bin_ind  << ", num_bins: " << num_bins << ", out_scores_i size: " << out_scores_i.size() << std::endl;
         }
 
       }

src/param/hbos_param.cpp

@@ -147,7 +147,7 @@ using namespace chimbuko;
  Histogram chimbuko::operator+(const Histogram& g, const Histogram& l) {
    Histogram combined;
    double min_runtime, max_runtime;
-   verboseStream << "Bin_Edges Size of Global Histogram: " << std::to_string(g.bin_edges().size()) << ", Bin_Edges Size of Local Histogram: " << std::to_string(l.bin_edges().size()) << std::endl;
+   //verboseStream << "Bin_Edges Size of Global Histogram: " << std::to_string(g.bin_edges().size()) << ", Bin_Edges Size of Local Histogram: " << std::to_string(l.bin_edges().size()) << std::endl;
    verboseStream << "Counts Size of Global Histogram: " << std::to_string(g.counts().size()) << ", Counts Size of Local Histogram: " << std::to_string(l.counts().size()) << std::endl;
 
    if (g.counts().size() <= 0) {
@@ -222,12 +222,17 @@ using namespace chimbuko;
          comb_binedges[1] = edge_val + bin_width;
        }
        else{
+	/*
          comb_binedges.resize(floor((max_runtime - min_runtime)/bin_width) + 2);
          for (int i = 0; i < comb_binedges.size(); i++) {
            comb_binedges[i] = edge_val;
            edge_val += bin_width;
          }
-
+	*/
+	for(edge_val = min_runtime; edge_val < max_runtime;) {
+	   comb_binedges.push_back(edge_val);
+           edge_val += bin_width;
+	}
        }
      }
 
@@ -282,7 +287,7 @@ using namespace chimbuko;
 
 
     *this = combined;
-    //this->set_hist_data(Histogram::Data(this->get_threshold(), this->counts(), this->bin_edges()));
+    this->set_hist_data(Histogram::Data(this->get_threshold(), this->counts(), this->bin_edges()));
     return *this;
  }
 
@@ -334,7 +339,7 @@ using namespace chimbuko;
    var = var / size;
    verboseStream << "Final Variance in _scott_binWidth: " << var << std::endl;
    std = sqrt(var);
-   verboseStream << "STD in _scott_binWidth: " << std << std::endl;
+   verboseStream << "STD in merging _scott_binWidth: " << std << std::endl;
    if (std <= 100.0) {return 0;}
 
    return ((3.5 * std ) / pow(size, 1/3));
@@ -370,10 +375,11 @@ using namespace chimbuko;
   // m_histogram.runtimes.push_back(x);
  //}
 
- void Histogram::create_histogram(const std::vector<double>& r_times)
+ int Histogram::create_histogram(const std::vector<double>& r_times)
  {
    std::vector<double> runtimes = r_times;
    const double bin_width = Histogram::_scott_binWidth(runtimes);
+   if (bin_width <= 0) {return -1;}
    std::sort(runtimes.begin(), runtimes.end());
    const int h = runtimes.size() - 1;
 
@@ -406,10 +412,10 @@ using namespace chimbuko;
      m_histogram.glob_threshold = min_threshold;
    }
    this->set_hist_data(Histogram::Data( m_histogram.glob_threshold, m_histogram.counts, m_histogram.bin_edges ));
-
+   return 0;
  }
 
- void Histogram::merge_histograms(const Histogram& g, const std::vector<double>& runtimes)
+ int Histogram::merge_histograms(const Histogram& g, const std::vector<double>& runtimes)
  {
 
    std::vector<double> r_times = runtimes;
@@ -421,10 +427,10 @@ using namespace chimbuko;
    }
 
    m_histogram.glob_threshold = g.get_threshold();
-   // verboseStream << "glob_threshold in merge_histograms = " << m_histogram.glob_threshold << std::endl;
-   this->create_histogram(r_times);
+   //verboseStream << "glob_threshold in merge_histograms = " << m_histogram.glob_threshold << std::endl;
+   return this->create_histogram(r_times);
    //this->set_hist_data(Histogram::Data( m_histogram.glob_threshold, m_histogram.counts, m_histogram.bin_edges ));
-
+   
  }
 
  nlohmann::json Histogram::get_json() const {