adding copod skewness and updating unit tests

Author: Sandeep Mittal

include/chimbuko/ad/ADOutlier.hpp

@@ -238,7 +238,7 @@ namespace chimbuko {
 
   private:
     double m_alpha; /**< Used to prevent log2 overflow */
-    double m_threshold; /**< Threshold used to filter anomalies in HBOS*/
+    double m_threshold; /**< Threshold used to filter anomalies in COPOD*/
     bool m_use_global_threshold; /**< Flag to use global threshold*/
     //double m_threshold; /** sync with global threshold */
     OutlierStatistic m_statistic; /** Which statistic to use for outlier detection */
@@ -253,13 +253,13 @@ namespace chimbuko {
   public:
 
     /**
-     * @brief Construct a new ADOutlierHBOS object
+     * @brief Construct a new ADOutlierCOPOD object
      *
      */
     ADOutlierCOPOD(OutlierStatistic stat = ExclusiveRuntime, double threshold = 0.99, bool use_global_threshold = true);
 
     /**
-     * @brief Destroy the ADOutlierHBOS object
+     * @brief Destroy the ADOutlierCOPOD object
      *
      */
     ~ADOutlierCOPOD();
@@ -308,11 +308,12 @@ namespace chimbuko {
 
   private:
     double m_alpha; /**< Used to prevent log2 overflow */
-    double m_threshold; /**< Threshold used to filter anomalies in HBOS*/
+    double m_threshold; /**< Threshold used to filter anomalies in COPOD*/
     bool m_use_global_threshold; /**< Flag to use global threshold*/
-    //double m_threshold; /** sync with global threshold */
-    OutlierStatistic m_statistic; /** Which statistic to use for outlier detection */
+    //double m_threshold; /**< sync with global threshold */
+    OutlierStatistic m_statistic; /**< Which statistic to use for outlier detection */
 
+    std::unordered_map<unsigned long, int> m_skewness; /**< skewness for each function*/
   };
 
 

src/ad/ADOutlier.cpp

@@ -571,6 +571,24 @@ Anomalies ADOutlierCOPOD::run(int step) {
 	    recoverable_error(std::string("AD: Zero function runtimes "));
 	    continue;
     }
+    verboseStream << "Size of runtimes: " << runtimes.size() << ", func_id: " << func_id << std::endl;
+    
+    //calculate skewness of runtimes for func_id
+    const double mu = std::accumulate(runtimes.begin(), runtimes.end(), 0.0) / runtimes.size();
+
+    std::vector<double> diff = std::vector<double>(runtimes.size());
+    std::transform(runtimes.begin(), runtimes.end(), diff.begin(), [mu](double x) {return x - mu;});
+    const double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
+    const double stdev = std::sqrt(sq_sum / runtimes.size());
+
+    double summ = 0;
+    for (int i=0; i<runtimes.size(); i++) {
+	summ += std::pow((runtimes.at(i) - mu), 3);
+    }
+
+    const double abs_skewness = summ / ((runtimes.size() - 1) * std::pow(stdev,3));
+    m_skewness[func_id] = (abs_skewness < 0) ? -1 : (abs_skewness > 0) ? 1 : 0;
+
   }
 
   //Update temp runstats to include information collected previously (synchronizes with the parameter server if connected)
@@ -637,34 +655,44 @@ unsigned long ADOutlierCOPOD::compute_outliers(Anomalies &outliers,
   verboseStream << "Size of empiricalCDF(recon_n_runtimes): " << func_n_ecdf.size() << std::endl;
 
   std::vector<double> mean_pn_ecdf = std::vector<double>(func_p_ecdf.size(), 0.0);
-  verboseStream << "Size of mean_pn_ecdf: " << mean_pn_ecdf.size() << std::endl;
+  verboseStream << "Size of mean_pn_ecdf: " << mean_pn_ecdf.size() << ", func_id: " << func_id << std::endl;
 
   for(int i=0; i < mean_pn_ecdf.size(); i++){
     mean_pn_ecdf.at(i) = (func_p_ecdf.at(i) + func_n_ecdf.at(i)) / 2.0;
     verboseStream << "mean_pn_ecdf.at(i): " << mean_pn_ecdf.at(i) << ", func_p_ecdf.at(i): " << func_p_ecdf.at(i) << ", func_n_ecdf.at(i): " << func_n_ecdf.at(i) << std::endl;
   }
 
+  //use skewness
+  std::vector<double> skewness_arr = std::vector<double>(func_p_ecdf.size(), 0.0);
+  const int p_sign = (m_skewness[func_id] - 1) < 0 ? -1 : (m_skewness[func_id] - 1) > 0 ? 1 : 0;
+  const int n_sign = (m_skewness[func_id] + 1) < 0 ? -1 : (m_skewness[func_id] + 1) > 0 ? 1 : 0;
+  
+  for (int i = 0; i< func_p_ecdf.size(); i++) {
+	skewness_arr.at(i) = (func_p_ecdf.at(i) * -1 * p_sign) + (func_n_ecdf.at(i) * n_sign);
+  	verboseStream << "skewness_arr.at(" << i << "): " << skewness_arr.at(i) << std::endl;
+  }
+
+  std::vector<double> final_comp = std::vector<double>(skewness_arr.size(), 0.0);
+  for (int i = 0; i < skewness_arr.size(); i++) {
+	final_comp.at(i) = std::max(skewness_arr.at(i), mean_pn_ecdf.at(i));
+  }
 
-  //for(int i=0; i < param[func_id].counts().size(); i++){
-  //  int count = param[func_id].counts().at(i);
-  //  double p = count / tot_runtimes;
-  //  prob_counts.at(i) += p;
-  //}
 
-  for(int i=0; i<mean_pn_ecdf.size(); i++)
-	  verboseStream << "mean_pn_ecdf at " << i << ": " << mean_pn_ecdf.at(i) << std::endl;
+  //for(int i=0; i<mean_pn_ecdf.size(); i++)
+	//  verboseStream << "mean_pn_ecdf at " << i << ": " << mean_pn_ecdf.at(i) << std::endl;
 
   //Create COPOD score vector
-  std::vector<double> out_scores_i = std::vector<double>(mean_pn_ecdf.size(), 0.0);
+  std::vector<double> out_scores_i = std::vector<double>(final_comp.size(), 0.0);
   verboseStream << "m_alpha: " << m_alpha << std::endl;
 
   double min_score = -1 * log2(0.0 + m_alpha);
-  double max_score = -1 * log2(1.0 + m_alpha);
+  double max_score = log2(1.0 + m_alpha) - min_score;
+  verboseStream << "Initializaing min_score: " << min_score << ", max_score: " << max_score <<std::endl;
   verboseStream << "out_scores_i: " << std::endl;
-  for(int i=0; i < mean_pn_ecdf.size(); i++){
-    double l = -1 * log2(mean_pn_ecdf.at(i) + m_alpha);
+  for(int i=0; i < final_comp.size(); i++){
+    double l = -1 * log2(final_comp.at(i) + m_alpha);
     out_scores_i.at(i) = l;
-    verboseStream << "Mean_ecdf at " << i << ": " << mean_pn_ecdf.at(i) << ", score: "<< l << std::endl;
+    verboseStream << "Final_comp at " << i << ": " << final_comp.at(i) << ", score: "<< l << std::endl;
     //if(prob_counts.at(i) > 0) {
       if(l < min_score){
         min_score = l;
@@ -683,9 +711,10 @@ unsigned long ADOutlierCOPOD::compute_outliers(Anomalies &outliers,
 
   //compute threshold
   verboseStream << "Global threshold before comparison with local threshold =  " << param[func_id].get_threshold() << std::endl;
-  double l_threshold = min_score + (m_threshold * (max_score - min_score));
+  double l_threshold = (max_score < 0) ? (-1 * m_threshold * (max_score - min_score)) : min_score + (m_threshold * (max_score - min_score));
+  verboseStream << "l_threshold computed: " << l_threshold << std::endl;
   if(m_use_global_threshold) {
-    if(l_threshold < param[func_id].get_threshold()) {
+    if(l_threshold < param[func_id].get_threshold() && param[func_id].get_threshold() > (-1 * log2(1.00001))) {
       l_threshold = param[func_id].get_threshold();
     } else {
       param[func_id].set_glob_threshold(l_threshold); //.get_histogram().glob_threshold = l_threshold;
@@ -694,31 +723,29 @@ unsigned long ADOutlierCOPOD::compute_outliers(Anomalies &outliers,
   }
 
   //Compute COPOD based score for each datapoint
-  //const double bin_width = param[func_id].bin_edges().at(1) - param[func_id].bin_edges().at(0);
-  //const int num_bins = param[func_id].counts().size();
-  //verboseStream << "Bin width: " << bin_width << std::endl;
 
   int top_out = 0;
-  //int running_idx = 0;
+  int running_idx = 0;
   for (auto itt : data) {
     if (itt->get_label() == 0) {
 
       const double runtime_i = this->getStatisticValue(*itt); //runtimes.push_back(this->getStatisticValue(*itt));
       double ad_score;
-      int running_idx = 0;
-
-      //find bin index of data(runtime_i) in merged histogram
-      for (int i=1; i < param[func_id].bin_edges().size(); i++) {
-	      if (runtime_i < param[func_id].bin_edges().at(i)) {
-		      running_idx = i-1;
-		      break;
-	      }
+      
+      if (running_idx < final_comp.size()) {
+      	verboseStream << "final_comp.at(" << running_idx << "): " << final_comp.at(running_idx) << ", func_id: " << func_id << ", runtime: " << runtime_i << std::endl;
+     
+        if (out_scores_i.at(running_idx) > l_threshold) //(final_comp.at(running_idx) > 0) // < 0.9)
+	        ad_score = l_threshold + 1;
+        else
+	        ad_score = l_threshold - 1;
+        
+	running_idx++;
+      }
+      else {
+	recoverable_error("AD: COPOD: runtime Index");
+        continue;
       }
-      verboseStream << "mean_pn_ecdf.at(running_idx): " << mean_pn_ecdf.at(running_idx) << std::endl;
-      if (mean_pn_ecdf.at(running_idx) < 0.99)
-	      ad_score = l_threshold + 1;
-      else
-	      ad_score = l_threshold - 1;
 
       itt->set_outlier_score(ad_score);
       verboseStream << "ad_score: " << ad_score << ", l_threshold: " << l_threshold << std::endl;

test/unit_tests/ad/COPODOutlierADs.cpp

@@ -28,7 +28,7 @@ class ADOutlierCOPODTest: public ADOutlierCOPOD{
 };
 
 
-TEST(HBOSADOutlierTestSyncParamWithPSComputeOutliers, Works){
+TEST(COPODADOutlierTestSyncParamWithPSComputeOutliers, Works){
   CopodParam global_params_ps; //parameters held in the parameter server
   CopodParam local_params_ad, local_params_ad2; //parameters collected by AD
 
@@ -49,7 +49,7 @@ TEST(HBOSADOutlierTestSyncParamWithPSComputeOutliers, Works){
     std::vector<double> runtimes;
     Histogram &r = local_params_ad[0];
     for(int i=0;i<N;i++) {
-      double val = i==N-1 ? 1000 : double(dist(gen));
+      double val = i==N-1 ? 10000 : double(dist(gen));
       call_list.push_back( createFuncExecData_t(0,0,0,  0, "my_func", 1000*(i+1), val) );
       runtimes.push_back(val);
       std::cout << "vals in localhist 1: " << val << std::endl;
@@ -65,7 +65,7 @@ TEST(HBOSADOutlierTestSyncParamWithPSComputeOutliers, Works){
     std::vector<double> runtimes;
     Histogram &r = local_params_ad2[0];
     for(int i=0;i<N;i++) {
-      double val = i==N-1 ? 1000 : double(dist(gen));
+      double val = i==N-1 ? 20000 : double(dist(gen));
       call_list2.push_back( createFuncExecData_t(0,0,0,  0, "my_func", 1000*(i+1), val) );
       runtimes.push_back(val);
       std::cout << "vals in localhist 2: " << val << std::endl;

test/unit_tests/ad/HBOSOutlierTestBPFile.cpp

@@ -115,7 +115,7 @@ TEST(HBOSADOutlierBPFileWithoutPServer, Works) {
     //Parameters for the connection to the instrumented binary trace output
     params.trace_engineType = "BPFile";  // BPFile or SST
     params.trace_data_dir = trace_data_dir; // *.bp location
-    std::string bp_prefix = "tau-metrics-xgc-es-cpp-gpu"; //"tau-metrics";  // bp file prefix (e.g. tau-metrics-[nwchem])
+    std::string bp_prefix = "tau-metrics"; //"tau-metrics-xgc-es-cpp-gpu"; // bp file prefix (e.g. tau-metrics-[nwchem])
 
     //The remainder are optional arguments. Enable using the appropriate command line switch
     params.program_idx = 0;

test/unit_tests/ad/SSTDOutlierTestBPFile.cpp

@@ -116,7 +116,7 @@ TEST(SSTDADOutlierBPFileWithoutPServer, Works) {
     //Parameters for the connection to the instrumented binary trace output
     params.trace_engineType = "BPFile"; // argv[1]; // BPFile or SST
     params.trace_data_dir = trace_data_dir; // *.bp location
-    std::string bp_prefix = "tau-metrics-xgc-es-cpp-gpu"; //"tau-metrics"; //argv[3]; // bp file prefix (e.g. tau-metrics-[nwchem])
+    std::string bp_prefix = "tau-metrics"; //"tau-metrics-xgc-es-cpp-gpu"; //argv[3]; // bp file prefix (e.g. tau-metrics-[nwchem])
 
     //The remainder are optional arguments. Enable using the appropriate command line switch
     params.program_idx = 0;