To main Chimbuko class, added the ability to pass a filename containing functions that are ignored by the AD

Added command line options to specify the above to driver and driver_multirank

Author: giltirn

app/driver.cpp

@@ -76,7 +76,8 @@ optionalArgsParser & getOptionalArgsParser(){
     addOptionalCommandLineArg(p, hbos_max_bins, "Set the maximum number of bins for histograms in the HBOS algorithm. Default 200.");
     addOptionalCommandLineArg(p, program_idx, "Set the index associated with the instrumented program. Use to label components of a workflow. (default 0)");
     addOptionalCommandLineArg(p, outlier_sigma, "Set the number of standard deviations that defines an anomalous event (default 6)");
-    addOptionalCommandLineArg(p, func_threshold_file, "A filename containing HBOS/COPOD algorithm threshold overrides for specified functions. Format is JSON: \"[ { \"fname\": <FUNC>, \"threshold\": <THRES> },... ]\". Empty string (default) uses default threshold for all funcs");
+    addOptionalCommandLineArg(p, func_threshold_file, "Provide the path to a file containing HBOS/COPOD algorithm threshold overrides for specified functions. Format is JSON: \"[ { \"fname\": <FUNC>, \"threshold\": <THRES> },... ]\". Empty string (default) uses default threshold for all funcs");
+    addOptionalCommandLineArg(p, ignored_func_file, "Provide the path to a file containing function names (one per line) which the AD algorithm will ignore. All such events are labeled as normal. Empty string (default) performs AD on all events.");    
     addOptionalCommandLineArg(p, net_recv_timeout, "Timeout (in ms) for blocking receives on client from parameter server (default 30000)");
     addOptionalCommandLineArg(p, pserver_addr, "Set the address of the parameter server. If empty (default) the pserver will not be used.");
     addOptionalCommandLineArg(p, hpserver_nthr, "Set the number of threads used by the hierarchical PS. This parameter is used to compute a port offset for the particular endpoint that this AD rank connects to (default 1)");

app/driver_multirank.cpp

@@ -58,6 +58,7 @@ optionalArgsParser & getOptionalArgsParser(){
     addOptionalCommandLineArg(p, program_idx, "Set the index associated with the instrumented program. Use to label components of a workflow. (default 0)");
     addOptionalCommandLineArg(p, outlier_sigma, "Set the number of standard deviations that defines an anomalous event (default 6)");
     addOptionalCommandLineArg(p, func_threshold_file, "A filename containing HBOS/COPOD algorithm threshold overrides for specified functions. Format is JSON: \"[ { \"fname\": <FUNC>, \"threshold\": <THRES> },... ]\". Empty string (default) uses default threshold for all funcs");
+    addOptionalCommandLineArg(p, ignored_func_file, "Provide the path to a file containing function names (one per line) which the AD algorithm will ignore. All such events are labeled as normal. Empty string (default) performs AD on all events.");
     addOptionalCommandLineArg(p, net_recv_timeout, "Timeout (in ms) for blocking receives on client from parameter server (default 30000)");
     addOptionalCommandLineArg(p, pserver_addr, "Set the address of the parameter server. If empty (default) the pserver will not be used.");
     addOptionalCommandLineArg(p, hpserver_nthr, "Set the number of threads used by the hierarchical PS. This parameter is used to compute a port offset for the particular endpoint that this AD rank connects to (default 1)");

include/chimbuko/chimbuko.hpp

@@ -26,6 +26,8 @@ namespace chimbuko {
 
     std::string func_threshold_file; /**< A filename containing HBOS/COPOD algorithm threshold overrides for specified functions. Format is JSON: "[ { "fname": <FUNC>, "threshold": <THRES> },... ]". Empty string (default) uses default threshold for all funcs*/ 
 
+    std::string ignored_func_file; /**< A filename containing function names (one per line) which the AD algorithm will ignore. All such events are labeled as normal. Empty string (default) performs AD on all events*/
+
     //Parameters associated with communicating with the parameter server*/
     std::string pserver_addr; /**< The address of the parameter server.
 				 < If no parameter server is in use, this string should be empty (length zero)

src/chimbuko.cpp

@@ -34,7 +34,8 @@ ChimbukoParams::ChimbukoParams(): rank(-1234),  //not set!
                                   analysis_step_freq(1),
                                   read_ignored_corrid_funcs(""),
                                   max_frames(-1),
-                                  func_threshold_file("")
+                                  func_threshold_file(""),
+                                  ignored_func_file("")
 {}
 
 void ChimbukoParams::print() const{
@@ -193,7 +194,10 @@ void Chimbuko::init_event(){
 	m_event->ignoreCorrelationIDsForFunction(func);
       }
       in.close();
+    }else{
+      fatal_error("Failed to open ignored-corried-func file " + m_params.read_ignored_corrid_funcs);
     }
+
   }
 }
 
@@ -239,6 +243,23 @@ void Chimbuko::init_outlier(){
   m_outlier->linkExecDataMap(m_event->getExecDataMap()); //link the map of function index to completed calls such that they can be tagged as outliers if appropriate
   if(m_net_client) m_outlier->linkNetworkClient(m_net_client);
   m_outlier->linkPerf(&m_perf);
+
+  //Read ignored functions
+  if(m_params.ignored_func_file.size()){
+    std::ifstream in(m_params.ignored_func_file);
+    if(in.is_open()) {
+      std::string func;
+      while (std::getline(in, func)){
+	headProgressStream(m_params.rank) << "Skipping anomaly detection for function \"" << func << "\"" << std::endl;
+	m_outlier->setIgnoreFunction(func);
+      }
+      in.close();
+    }else{
+      fatal_error("Failed to open ignored-func file " + m_params.ignored_func_file);
+    }
+  }
+
+
 }
 
 void Chimbuko::init_counter(){