Added a optionalCommandLineArgBase derived class to AlgoParams allowing it to be constructed automatically by giving a filename through the usual cmdline parser tools

AD algorithms now have a function to return their name/type as a string
Moved the algorithm parameters into an AlgoParams instance inside the Chimbuko base class params struct
Passing of algorithm parameters to the Chimbuko driver is now performed by passing a single argument of the filename to the AlgoParams file. At the moment this JSON file is built by run_services to avoid changing the usage pattern

Author: giltirn

include/chimbuko/core/ad/ADOutlier.hpp

@@ -11,6 +11,7 @@
 #include "chimbuko/core/ad/ADNetClient.hpp"
 #include "chimbuko/core/util/PerfStats.hpp"
 #include "chimbuko/core/ad/ADDataInterface.hpp"
+#include "chimbuko/core/util/commandLineParser.hpp"
 
 namespace chimbuko { 
   /**
@@ -57,6 +58,31 @@ namespace chimbuko {
        * @brief Equivalence operator
        */
       bool operator==(const AlgoParams &r) const;
+
+      /**
+       * @brief Parser object that reads the data from a json file with provided filename
+       */
+      class cmdlineParser: public optionalCommandLineArgBase{
+      private:
+	std::string m_arg; /**< The argument, format "-a" */
+	std::string m_help_str; /**< The help string */
+	AlgoParams &member;
+      public:
+	cmdlineParser(AlgoParams &member, const std::string &arg, const std::string &help_str): m_arg(arg), m_help_str(help_str), member(member){}
+
+	/**
+	 * @brief If the first string matches the internal arg string (eg "-help"), a number of strings are consumed from the array 'vals' and that number returned. 
+	 * A value of -1 indicates the argument did not match.
+	 *
+	 * @param vals An array of strings
+	 * @param vals_size The length of the string array
+	 */
+	int parse(const std::string &arg, const char** vals, const int vals_size) override;
+	/**
+	 * @brief Print the help string for this argument to the ostream
+	 */
+	void help(std::ostream &os) const override;
+      };
     };
 
 
@@ -120,6 +146,10 @@ namespace chimbuko {
      */
     void setGlobalModelSyncFrequency(int to){ m_global_model_sync_freq = to; }
 
+    /**
+     * @brief Return the algorithm name
+     */
+    virtual std::string getAlgorithmName() const = 0;
   protected:
     /** @brief Synchronize the input model with the global model    
      *
@@ -176,6 +206,11 @@ namespace chimbuko {
 
     void run(ADDataInterface &data, int step=0) override;
 
+    /**
+     * @brief Return the algorithm name
+     */
+    std::string getAlgorithmName() const override{ return "sstd"; }
+
   protected:
 
     /**
@@ -234,6 +269,11 @@ namespace chimbuko {
 
     void run(ADDataInterface &data, int step=0) override;
 
+    /**
+     * @brief Return the algorithm name
+     */
+    std::string getAlgorithmName() const override{ return "hbos"; }
+
     /**
      * @brief Override the default threshold for a particular function
      * @param func The function name
@@ -307,6 +347,11 @@ namespace chimbuko {
 
     void run(ADDataInterface &data, int step=0) override;
 
+    /**
+     * @brief Return the algorithm name
+     */
+    std::string getAlgorithmName() const override{ return "copod"; }
+
     /**
      * @brief Override the default threshold for a particular function
      * @param func The function name

include/chimbuko/core/chimbuko.hpp

@@ -24,16 +24,9 @@ namespace chimbuko {
     int prov_record_stopstep; /**< If != -1, the IO step on which to stop recording provenance information for anomalies */
     int prov_io_freq; /**< The frequency, in steps, at which provenance data is written/sent to the provDB. For steps between it is buffered.*/
 
-
     //Parameters associated with the outlier detection algorithm
-    std::string ad_algorithm; /**< Algorithm for Anomaly Detection. Set in config file*/
-
-    double outlier_sigma; /**< The number of sigma (standard deviations) away from the mean runtime for an event to be considered anomalous */
-
-    double hbos_threshold; /**< Threshold used by HBOS algorithm to filter outliers. Set in config file*/
-    bool hbos_use_global_threshold; /**< Global threshold flag in HBOS*/
-    int hbos_max_bins; /**< Maximum number of bins to use in HBOS algorithm histograms*/
-
+    ADOutlier::AlgoParams algo_params; /**< The algorithm parameters */
+    
     int global_model_sync_freq; /**< How often (in steps) the global model is updated (default 1)*/
 
     //Parameters associated with communicating with the parameter server

scripts/launch/run_services.sh

@@ -78,6 +78,17 @@ echo "==========================================="
 echo "Starting Chimbuko services with module: ${module}"
 echo "==========================================="
 
+#Generate the AlgoParams JSON script from the inputs
+algo_params_file=${var_dir}/algo_params.json
+cat <<EOF > ${algo_params_file}
+{
+  "algorithm" : "${ad_alg}",
+  "glob_thres" : true,
+  "hbos_max_bins" : 200,
+  "hbos_thres" : ${ad_outlier_hbos_threshold},
+  "sstd_sigma" : ${ad_outlier_sstd_sigma}
+}
+EOF
 
 #Provenance database
 extra_args=${ad_extra_args}
@@ -445,14 +456,11 @@ fi
 #echo "Chimbuko Services: Processes are: " $(ps)
 
 #Check that the variables passed to the AD from the config file are defined
-testit=${ad_outlier_sstd_sigma}
 testit=${ad_win_size}
-testit=${ad_alg}
-testit=${ad_outlier_hbos_threshold}
 
 ############################################
 #Generate the command to launch the AD module
-ad_opts="${extra_args} -err_outputpath ${log_dir} -outlier_sigma ${ad_outlier_sstd_sigma} -anom_win_size ${ad_win_size} -ad_algorithm ${ad_alg} -hbos_threshold ${ad_outlier_hbos_threshold}"
+ad_opts="${extra_args} -err_outputpath ${log_dir} -algo_params_file ${algo_params_file} -anom_win_size ${ad_win_size}"
 
 if [[ "$(declare -p EXE_NAME)" =~ "declare -a" ]]; then
     echo "The user has specified a workflow comprising multiple components. Chimbuko will generate separate files containing pre-generated launch commands."

src/core/ad/ADOutlier.cpp

@@ -61,6 +61,25 @@ nlohmann::json ADOutlier::AlgoParams::getJson() const{
 #undef JSON_SET
 }
 
+int ADOutlier::AlgoParams::cmdlineParser::parse(const std::string &arg, const char** vals, const int vals_size){
+  std::cout << "TEST ARG " << arg << "==" << m_arg << std::endl;
+  if(arg == m_arg){
+    std::cout << "TEST ARG FOUND ARG " << arg << std::endl;
+    if(vals_size < 1) return -1;
+
+    try{
+      member.loadJsonFile(vals[0]);
+    }catch(const std::exception &exc){
+      return -1;
+    }
+    return 1;
+  }
+  return -1;
+}
+void ADOutlier::AlgoParams::cmdlineParser::help(std::ostream &os) const{
+  os << m_arg << " : " << m_help_str;
+}
+
 
 
 /* ---------------------------------------------------------------------------

src/core/ad/ADcmdLineArgs.cpp

@@ -1,15 +1,11 @@
 #include<chimbuko/core/ad/ADcmdLineArgs.hpp>
+#include<chimbuko/core/ad/ADOutlier.hpp>
 #include<chimbuko/core/verbose.hpp>
 
 using namespace chimbuko;
 
 void chimbuko::setupBaseOptionalArgs(commandLineParser &parser, ChimbukoBaseParams &into){
-  addOptionalCommandLineArgWithDefault(parser, into, ad_algorithm, "hbos", "Set an AD algorithm to use: hbos or sstd (default \"hbos\").");
-  addOptionalCommandLineArgWithDefault(parser, into, hbos_threshold, 0.99, "Set Threshold for HBOS anomaly detection filter (default 0.99).");
-  addOptionalCommandLineArgWithDefault(parser, into, hbos_use_global_threshold, true, "Set true to use a global threshold in HBOS algorithm (default true).");
-  addOptionalCommandLineArgWithDefault(parser, into, hbos_max_bins, 200, "Set the maximum number of bins for histograms in the HBOS algorithm (default 200).");
   addOptionalCommandLineArgOptArgWithDefault(parser, into, ana_obj_idx, program_idx, 0, "Set the index associated with the instrumented program. Use to label components of a workflow. (default 0)");
-  addOptionalCommandLineArgWithDefault(parser, into, outlier_sigma, 6.0, "Set the number of standard deviations that defines an anomalous event (default 6)");
   addOptionalCommandLineArgWithDefault(parser, into, net_recv_timeout, 30000, "Timeout (in ms) for blocking receives on client from parameter server (default 30000)");
   addOptionalCommandLineArgWithDefault(parser, into, pserver_addr, "", "Set the address of the parameter server. If empty (default) the pserver will not be used.");
   addOptionalCommandLineArgWithDefault(parser, into, hpserver_nthr, 1, "Set the number of threads used by the hierarchical PS. This parameter is used to compute a port offset for the particular endpoint that this AD rank connects to (default 1)");
@@ -39,4 +35,6 @@ void chimbuko::setupBaseOptionalArgs(commandLineParser &parser, ChimbukoBasePara
   addOptionalCommandLineArgWithDefault(parser, into, prov_io_freq, 1, "Set the frequency in steps at which provenance data is writen/sent to the provDB (default 1)");
   addOptionalCommandLineArgWithDefault(parser, into, analysis_step_freq, 1, "Set the frequency in IO steps between analyzing the data. Data will be accumulated over intermediate steps. (default 1)");
   parser.addOptionalArg(progressHeadRank()=0, "-logging_head_rank", "Set the head rank upon which progress logging will be output (default 0)");
+
+  parser.addOptionalArg(new ADOutlier::AlgoParams::cmdlineParser(into.algo_params, "-algo_params_file", "Set the filename containing the algorithm name and hyperparameters."));
 }

src/core/chimbuko.cpp

@@ -12,13 +12,8 @@ using namespace chimbuko;
 ChimbukoBaseParams::ChimbukoBaseParams(): rank(-1234),  //not set!
 				  ana_obj_idx(0),
 				  verbose(true),
-				  outlier_sigma(6.),
                                   net_recv_timeout(30000),
 				  pserver_addr(""), hpserver_nthr(1),
-				  ad_algorithm("hbos"),
-				  hbos_threshold(0.99),
-				  hbos_use_global_threshold(true),
-				  hbos_max_bins(200),
 #ifdef ENABLE_PROVDB
 				  provdb_addr_dir(""), nprovdb_shards(1), nprovdb_instances(1), provdb_mercury_auth_key(""),
 #endif
@@ -38,15 +33,12 @@ ChimbukoBaseParams::ChimbukoBaseParams(): rank(-1234),  //not set!
 
 
 void ChimbukoBaseParams::print() const{
-  std::cout << "AD Algorithm: " << ad_algorithm
+  std::cout << "AD Algorithm: " << algo_params.algorithm
 	    << "\nAnalysis Objective Idx: " << ana_obj_idx
 	    << "\nRank       : " << rank
 #ifdef _USE_ZMQNET
 	    << "\nPS Addr    : " << pserver_addr
 #endif
-	    << "\nSigma      : " << outlier_sigma
-	    << "\nHBOS/COPOD Threshold: " << hbos_threshold
-	    << "\nUsing Global threshold: " << hbos_use_global_threshold
 	    << "\nInterval   : " << interval_msec << " msec"
             << "\nNetClient Receive Timeout : " << net_recv_timeout << "msec"
 	    << "\nPerf. metric outpath : " << perf_outputpath
@@ -143,15 +135,10 @@ void ChimbukoBase::init_net_client(){
 
 
 void ChimbukoBase::init_outlier(){
-  ADOutlier::AlgoParams params;
-  params.algorithm = m_base_params.ad_algorithm;
-  params.hbos_thres = m_base_params.hbos_threshold;
-  params.glob_thres = m_base_params.hbos_use_global_threshold;
-  params.sstd_sigma = m_base_params.outlier_sigma;
-  params.hbos_max_bins = m_base_params.hbos_max_bins;
+  headProgressStream(m_base_params.rank) << "driver rank " << m_base_params.rank << " initializing outlier algorith with params:\n" << m_base_params.algo_params.getJson().dump(4) << std::endl;
   //params.func_threshold_file = m_base_params.func_threshold_file;
 
-  m_outlier = ADOutlier::set_algorithm(m_base_params.rank, params);
+  m_outlier = ADOutlier::set_algorithm(m_base_params.rank, m_base_params.algo_params);
   if(m_net_client) m_outlier->linkNetworkClient(m_net_client);
   m_outlier->linkPerf(&m_perf);
   m_outlier->setGlobalModelSyncFrequency(m_base_params.global_model_sync_freq);

src/modules/performance_analysis/chimbuko.cpp

@@ -448,7 +448,7 @@ bool Chimbuko::readStep(std::unique_ptr<ADDataInterface> &iface){
     else{ fatal_error("Invalid statistic"); }
 
     ADExecDataInterface *data_iface = new ADExecDataInterface(m_event->getExecDataMap(), stat);
-    if( (this->getBaseParams().ad_algorithm == "sst" || this->getBaseParams().ad_algorithm == "SST") && std::getenv("CHIMBUKO_DISABLE_CUDA_JIT_WORKAROUND") == nullptr )
+    if( this->getAD().getAlgorithmName() == "sstd" && std::getenv("CHIMBUKO_DISABLE_CUDA_JIT_WORKAROUND") == nullptr )
       data_iface->setIgnoreFirstFunctionCall(&m_func_seen);
     iface.reset(data_iface);
   }

test/mainADwithProvDB.cpp

@@ -28,9 +28,9 @@ TEST(ADTestWithProvDB, BpfileTest)
 
     params.base_params.prov_outputpath = ""; //don't output
 
-    params.base_params.outlier_sigma = 6.0;
+    params.base_params.algo_params.sstd_sigma = 6.0;
     params.base_params.only_one_frame = true; //just analyze first IO frame
-    params.base_params.ad_algorithm = "sstd";
+    params.base_params.algo_params.algorithm = "sstd";
 
     params.base_params.provdb_addr_dir = addr_file_dir;
     params.base_params.nprovdb_shards = nshards;

test/test_ad.cpp

@@ -76,12 +76,13 @@ TEST_F(ADTest, BpfileTest)
 
     params.base_params.prov_outputpath = "./temp"; //have to output somewhere if provdb not in use
 
-    params.base_params.outlier_sigma = 6.0;
+    params.base_params.algo_params.sstd_sigma = 6.0;
+    params.base_params.algo_params.algorithm = "sstd";
     params.base_params.only_one_frame = true; //just analyze first IO frame
 
     params.anom_win_size = 0;
 
-    params.base_params.ad_algorithm = "sstd";
+
 
     Chimbuko driver;
     int step;
@@ -249,10 +250,10 @@ TEST_F(ADTest, BpfileWithNetTest)
     params.base_params.perf_step = 1;
 #endif
 
-    params.base_params.only_one_frame = true;
-    params.base_params.outlier_sigma = 6.0;
+    params.base_params.algo_params.sstd_sigma = 6.0;
+    params.base_params.algo_params.algorithm = "sstd";
 
-    params.base_params.ad_algorithm = "sstd";
+    params.base_params.only_one_frame = true;
 
     Chimbuko driver;