To ADsim, added the ability to apply an AD algorithm to the input trace data rather than user tagging

Added an ADsim example using a real AD algorithm and the ADsim API
ADsim event indices now use start step rather than end step in id

Author: giltirn

sim/include/sim/ad.hpp

@@ -5,6 +5,7 @@
 #include<chimbuko/ad/ADNormalEventProvenance.hpp>
 #include<chimbuko/ad/ADEvent.hpp>
 #include<chimbuko/ad/ADCounter.hpp>
+#include<chimbuko/ad/ADOutlier.hpp>
 #include<map>
 
 namespace chimbuko_sim{
@@ -17,6 +18,7 @@ namespace chimbuko_sim{
     std::unordered_map<unsigned long, std::list<ExecData_t> > m_all_execs; /**< Map of thread to execs */
     std::unordered_map<eventID, CallListIterator_t> m_eventid_map; /**< Map from event index to iterator */
     std::map<unsigned long, std::list<CallListIterator_t> > m_step_execs; /**< Events in any given step */
+    std::map<unsigned long, unsigned long> m_entry_step_count; /**< Count of events starting on a given step (used to assign event indices) */
 
     unsigned long m_largest_step; /**< Largest step index of an inserted function execution thus far encountered */
     unsigned long m_program_start;
@@ -28,6 +30,8 @@ namespace chimbuko_sim{
     ADNormalEventProvenance m_normal_events;
     ADMetadataParser m_metadata;
     std::unique_ptr<ADProvenanceDBclient> m_pdb_client;
+    ADLocalNetClient m_net_client; /**< The local net client. Only activated if an AD algorithm is used */
+    ADOutlier* m_outlier; /**< The local outlier algorithm instance, if used*/
   public:
     /** 
      * @brief Initialize the AD simulator
@@ -50,7 +54,18 @@ namespace chimbuko_sim{
     ADsim(int window_size, int pid, int rid, unsigned long program_start, unsigned long step_freq): ADsim(){
       init(window_size, pid, rid, program_start, step_freq);
     }
-    ADsim(){}
+    ADsim(): m_outlier(nullptr){}
+
+    ADsim(ADsim &&r);
+    
+    ~ADsim(){ if(m_outlier) delete m_outlier; }
+
+    /**
+     * @brief Set up the AD algorithm. Parameters correspond to those in ADOutlier::set_algorithm
+     *
+     * Using an AD algorithm to analyze the data is optional. Set the ad_algorithm() string to a value other than "none" to switch the algorithm
+     */
+    void setupADalgorithm(const double hbos_thres, const bool glob_thres, const double sstd_sigma, ADOutlier::OutlierStatistic stat = ADOutlier::ExclusiveRuntime);
 
     ADProvenanceDBclient &getProvDBclient(){ return *m_pdb_client; }
 
@@ -69,7 +84,7 @@ namespace chimbuko_sim{
 			       const std::string &func_name,
 			       unsigned long start,
 			       unsigned long runtime,
-			       bool is_anomaly,
+			       bool is_anomaly = false,
 			       double outlier_score = 0.);
 
     /**

sim/include/sim/ad_params.hpp

@@ -1,11 +1,42 @@
 #include<chimbuko/param/sstd_param.hpp>
+#include<chimbuko/ad/ADOutlier.hpp>
 #include "id_map.hpp"
 
 namespace chimbuko_sim{
   using namespace chimbuko;
 
+  struct ADalgParams{
+    std::string algorithm;
+    double hbos_thres;
+    bool glob_thres;
+    double sstd_sigma;
+    ADOutlier::OutlierStatistic stat;
+    
+    ADalgParams(const std::string &algorithm, double hbos_thres, bool glob_thres, double sstd_sigma, ADOutlier::OutlierStatistic stat): algorithm(algorithm), hbos_thres(hbos_thres), glob_thres(glob_thres), sstd_sigma(sstd_sigma), stat(stat){}
+
+    /**
+     * @brief Default parameters
+     */
+    ADalgParams(): algorithm("none"), hbos_thres(0.99), glob_thres(true), sstd_sigma(12.0), stat(ADOutlier::ExclusiveRuntime){}
+  };
+  
+  /**
+   * @brief Set the parameters of the ad algorithm
+   *
+   * If algorithm=="none" (defulat) the user is expected to tag events manually (the params put in the provDB will use the SSTD algorithm format), otherwise it will use one of Chimbuko's algorithms
+   */ 
+  inline ADalgParams & adAlgorithmParams(){ static ADalgParams alg; return alg; }
+
+
+  /**
+   * @brief If an AD algorithm is not used (ad_algorithm() == "none" [default]) this is the object containing the params that are placed in the provDB
+   */
   inline SstdParam & globalParams(){ static SstdParam p; return p; }
 
-  //"register" a function, generating fake statistics given a provided mean and standard deviation
+  /**
+   * @brief If an AD algorithm is not used (ad_algorithm() == "none" [default]) all functions should be 'registered' using this command
+   *
+   * This function sets up both the function idx map and enters the provided statistics in the globalParams() instance
+   */
   void registerFunc(const std::string &func_name, unsigned long mean_runtime, unsigned long std_dev_runtime, int seen_count);
 };

sim/include/sim/id_map.hpp

@@ -4,10 +4,22 @@
 #include<unordered_map>
 
 namespace chimbuko_sim{
-  //All functions must be registered
+  /**
+   * @brief Map of function name to index
+   */
   std::unordered_map<std::string, unsigned long> & funcIdxMap();
 
-  //Automatically register counter names with indices
+  /**
+   * @brief Register a function (set up the function idx map)
+   * @param func_name The function name
+   * @return The assigned function index
+   */
+  unsigned long registerFunc(const std::string &func_name);
+
+
+  /**
+   * @brief A class to automatically register counter names with indices
+   */
   class CounterIdxManager{
     std::unordered_map<std::string, unsigned long> cid_map;
     std::unordered_map<int, std::string> cname_map;
@@ -21,5 +33,8 @@ namespace chimbuko_sim{
     const std::unordered_map<int, std::string>* getCounterMap(){ return &cname_map; }
   };
 
+  /**
+   * @brief Get the global shared instance of the counter index manager
+   */
   inline CounterIdxManager & getCidxManager(){ static CounterIdxManager cman; return cman; }
 };

sim/include/sim/pserver.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include<chimbuko/param.hpp>
 #include<chimbuko/pserver/global_anomaly_stats.hpp>
 #include<chimbuko/pserver/global_counter_stats.hpp>
 
@@ -12,23 +13,40 @@ namespace chimbuko_sim{
     GlobalCounterStats global_counter_stats; //global counter statistics
     PSstatSenderGlobalAnomalyStatsPayload anomaly_stats_payload;
     PSstatSenderGlobalCounterStatsPayload counter_stats_payload;
+    LocalNet m_net; /**< The net server */
+    ParamInterface* m_ad_params; /**< If using an AD algorithm to analyze the trace, these are the global parameters*/
   public:
+    
+    /**
+     * @brief Construct the pserver. To use an AD algorithm, the static function ad_algorithm should be changed *before* constructing the pserver
+     */
+    pserverSim();
 
-    pserverSim(): anomaly_stats_payload(&global_func_stats), counter_stats_payload(&global_counter_stats){}
+    ~pserverSim();
 
-    //Stand-in for the func statistics comms from AD -> pserver
+    /**
+     * @brief Stand-in for the func statistics comms from AD -> pserver
+     */
     void addAnomalyData(const ADLocalFuncStatistics &loc){
       global_func_stats.add_anomaly_data(loc);
     }
+
+    /**
+     * @brief Stand-in for the counter statistics comms from AD -> pserver
+     */
     void addCounterData(const ADLocalCounterStatistics &loc){
       global_counter_stats.add_counter_data(loc);
     }
-
-    //Mirror write functionality of PSstatSender
+    
+    /**
+     * @brief Mirror write functionality of PSstatSender
+     */
     void writeStreamingOutput() const;
   };  
 
-
+  /**
+   * @brief Get the shared pserver instance
+   */
   inline pserverSim & getPserver(){ static pserverSim ps; return ps; }
 
 };

sim/main/Makefile.am

@@ -2,7 +2,10 @@ AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/3rdparty -I$(top_srcdir)/s
 LDADD = ../src/libchimbuko_sim.la ../../src/libchimbuko.la -lstdc++fs 
 
 simmaindir = $(prefix)/sim/main
-simmain_PROGRAMS = example1
+simmain_PROGRAMS = example1 example2
 
 example1_SOURCES = example1.cpp
 example1_LDADD = $(LDADD)
+
+example2_SOURCES = example2.cpp
+example2_LDADD = $(LDADD)

sim/main/example2.cpp

@@ -0,0 +1,77 @@
+//This example demonstrates the simulator using the ADSim API directly, and using a real outlier algorithm
+#include<mpi.h>
+#include<sim.hpp>
+#include<random>
+
+using namespace chimbuko_sim;
+
+int main(int argc, char **argv){
+  MPI_Init(&argc, &argv);
+
+  int window_size = 5; //number of events to record around an anomaly in the provenance data
+  int pid = 0; //program index
+  unsigned long program_start = 100;
+  unsigned long step_freq = 1000;
+
+  //Set the AD algorithm *before instantiating the simulator!
+  ADalgParams & alg = adAlgorithmParams();
+  alg.algorithm = "sstd";
+  alg.sstd_sigma = 12.0;
+  
+  //Setup the "AD" instances
+  int n_ranks = 1;
+  std::vector<ADsim> ad;
+  for(int r=0;r<n_ranks;r++)
+    ad.push_back(ADsim(window_size, pid, r, program_start, step_freq));
+
+  //Setup some functions
+  registerFunc("main");
+  registerFunc("child");
+
+  //Setup some systems
+  std::vector<int> cpu_threads(4); for(int i=0;i<4;i++) cpu_threads[i] = i;
+
+  int gpus = 2;
+  std::vector<int> gpu_threads(gpus); //assign virtual thread indices to gpus and register
+  for(int g=0;g<gpus;g++){    
+    gpu_threads[g] = cpu_threads.size() + g; //GPU virtual index should not correspond with a CPU core thread index
+    ad[0].registerGPUthread(gpu_threads[g]);
+  }
+
+  std::default_random_engine gen;
+  std::normal_distribution<double> dist(100,20); //normal std.dev = 20
+
+  unsigned long child_start = 200;
+  int nexec = 500;
+  std::vector<int> starts(nexec);
+  std::vector<int> runtimes(nexec);
+  for(int i=0;i<nexec;i++){
+    if(i==0) starts[i] = child_start;
+    else starts[i] = starts[i-1] + runtimes[i-1] + 1;
+    runtimes[i] = int( dist(gen) );
+  }
+  //Add an anomaly
+  starts.push_back( starts.back() + runtimes.back() + 1 );
+  runtimes.push_back( 400 );    
+  nexec++;
+
+  int program_end = starts.back() + runtimes.back() + 50;
+
+  auto main = ad[0].addExec(0, "main", program_start, program_end - program_start);
+  for(int i=0;i<nexec;i++){
+    CallListIterator_t child = ad[0].addExec(0, "child", starts[i], runtimes[i]);
+    ad[0].bindParentChild(main, child);
+  }
+
+  //Run the simulation
+  int nstep = ad[0].largest_step() + 1;
+  std::cout << "Got data for " << nstep << " steps" << std::endl;
+
+  for(int i=0;i<nstep;i++){
+    std::cout << "Step " << i << " contains " << ad[0].nStepExecs(i) << " execs" << std::endl;
+    ad[0].step(i);
+  }
+
+  MPI_Finalize();
+  return 0;
+}