SSTD algorithm score is now the relative number of standard deviations from mean rather than a p-value, as all non-trivial outliers have a p-value that is 0 to many sig figs!

Added a "severity" score to the outliers in the provenance database (currently the anomaly runtime) that allows for sorting anomalies by impact on application
Added function to simulated AD to update the runtime on an existing exec, allowing for easier trace generation for nested functions
Like the real pserver, the simulated pserver now only outputs stats data in each period if any data has been collected. Output data is now "pretty printed".
Added a new example simulator main program with manually tagged anomalies for a multi-thread, multi-rank program
Added a perl script to locate unmatched correlation IDs in a sst_view dump on an ADIOS2 binary

Author: giltirn

scripts/find_unmatched_corids.pl

@@ -0,0 +1,64 @@
+#!/usr/bin/perl
+
+$ARGC=scalar @ARGV;
+if($ARGC != 1){
+    print "Need dump of ADIOS2 binary";
+    exit(-1);
+}
+
+
+
+open(IN,$ARGV[0]);
+$idx=-1;
+$active=0;
+$ndata=-1;
+$i=0;
+
+%corid;
+
+while(<IN>){
+    $line=$_;
+    if($line=~m/counter\s(\d+)\s.*Correlation ID/){
+	$idx=$1;
+	print "Found counter index $idx\n";
+    }elsif($line=~m/counter_values.*Parsed size (\d+)/){
+	$active=1;
+	$ndata=$1;
+	$i=0;
+	print "Found counter data block of size $ndata\n";
+    }elsif($active == 1){
+	if($line=~m/3 \):${idx} \(\d+ 4 \):(\d+)/){
+	    if(exists $corid{$1}){
+		delete $corid{$1};
+		print "Matched $1\n";
+	    }else{
+		$corid{$1} = 1;
+		print "Added $1\n";
+	    }
+		
+	    
+	    #print "$i $idx $1\n";
+	    
+	}
+	    #counter_values {AvailableStepsCount:15 Max:1627915315912147 Min:0 Shape:188, 6 SingleValue:false Type:uint64_t } Parsed size 188 6
+	
+
+	$i++;
+	if($i == $ndata){
+	    print "Ending data block on line $i\n";
+	    $active=0;
+	}
+    }
+    
+    #counter 0 {Elements:1 Type:string Value:"Correlation ID" }
+
+    #/bld/benchmark_suite/cupti_gpu_kernel_outlier/chimbuko/adios2/data
+}
+close(IN);
+
+$nunmatched=scalar keys %corid;
+print "--------$nunmatched unmatched corids\n";
+
+foreach $id (keys %corid){
+    print "--------Unmatched $id\n";
+}

sim/include/sim/ad.hpp

@@ -90,6 +90,15 @@ namespace chimbuko_sim{
 			       bool is_anomaly = false,
 			       double outlier_score = 0.);
 
+    /**
+     * @brief Modify the runtime of an existing execution
+     * @param exec_it The iterator to the execution (output of previous addExec)
+     * @param new_runtime The new function runtime
+     */
+    void updateRuntime(CallListIterator_t exec_it,
+		       unsigned long new_runtime);
+
+
     /**
      * @brief Attach a counter to an execution t_delta us after the start of the execution
      * @param counter_name The counter name

sim/main/Makefile.am

@@ -2,10 +2,13 @@ AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/3rdparty -I$(top_srcdir)/s
 LDADD = ../src/libchimbuko_sim.la ../../src/libchimbuko.la -lstdc++fs 
 
 simmaindir = $(prefix)/sim/main
-simmain_PROGRAMS = example1 example2
+simmain_PROGRAMS = example1 example2 example3
 
 example1_SOURCES = example1.cpp
 example1_LDADD = $(LDADD)
 
 example2_SOURCES = example2.cpp
 example2_LDADD = $(LDADD)
+
+example3_SOURCES = example3.cpp
+example3_LDADD = $(LDADD)

sim/main/example3.cpp

@@ -0,0 +1,88 @@
+//This example demonstrates the simulator using the ADSim API directly with multiple ranks and multiple threads per rank
+//Anomalies are explicitly tagged
+#include<mpi.h>
+#include<sim.hpp>
+#include<random>
+
+using namespace chimbuko_sim;
+
+int main(int argc, char **argv){
+  MPI_Init(&argc, &argv);
+
+  int window_size = 5; //number of events to record around an anomaly in the provenance data
+  int pid = 0; //program index
+  unsigned long program_start = 100;
+  unsigned long step_freq = 1000;
+ 
+  //Setup the "AD" instances
+  int n_ranks = 4;
+  int threads_per_rank = 2;
+
+  std::vector<ADsim> ad;
+  for(int r=0;r<n_ranks;r++)
+    ad.push_back(ADsim(window_size, pid, r, program_start, step_freq));
+
+  //Setup some functions
+  registerFunc("main", 500, 50, 100);
+  registerFunc("child", 250, 25, 100);
+
+  int anom_runtimes[3] = {500,750,1000};
+  double anom_scores[3] = {2,3,4};
+
+  //Setup some traces
+  int nexec = 500;
+  int anom_freq = 50;
+    
+  for(int rank=0;rank<n_ranks;rank++){
+    for(int thread=0;thread<threads_per_rank;thread++){
+      int anom_off = 0;
+
+      unsigned long child_start = 200;
+
+      unsigned long time = program_start;
+      auto main = ad[rank].addExec(thread, "main", time, 0); //will update the end time later
+      for(int i=0;i<nexec;i++){
+	CallListIterator_t child;
+	unsigned long runtime;
+	if(i!=0 && i % anom_freq == 0){ //an anomaly!
+	  runtime = anom_runtimes[anom_off];
+	  double score = anom_scores[anom_off];	 
+	  child = ad[rank].addExec(thread, "child", time, runtime, true, score);
+	  
+	  std::cout << rank << " " << thread << " " << i << " anomaly " << runtime << std::endl;
+	  
+	  anom_off = (anom_off + 1) % 3; //cycle through 3 times
+	}else{
+	  runtime = 250;
+	  child = ad[rank].addExec(thread, "child", time, runtime);
+
+	  std::cout << rank << " " << thread << " " << i << " normal " << runtime << std::endl;
+	}
+	ad[rank].bindParentChild(main, child);
+
+	time = time + runtime + 1;
+      }
+      ad[rank].updateRuntime(main, time - program_start);
+    }
+  }
+
+  //Run the simulation
+  int nstep = ad[0].largest_step() + 1;
+  for(int r=1;r<n_ranks;r++)
+    nstep = std::max( (unsigned long)nstep, ad[r].largest_step() + 1);
+  
+  std::cout << "Got data for " << nstep << " steps" << std::endl;
+
+  for(int i=0;i<nstep;i++){
+    for(int r=0;r<n_ranks;r++){
+      std::cout << "Rank " << r << " step " << i << " contains " << ad[r].nStepExecs(i) << " execs" << std::endl;
+      ad[r].step(i);
+    }
+    //PServer dump its output
+    getPserver().writeStreamingOutput();
+  }
+
+  MPI_Finalize();
+  std::cout << "Done" << std::endl;
+  return 0;
+}

sim/src/ad.cpp

@@ -104,6 +104,56 @@ CallListIterator_t ADsim::addExec(const int thread,
 }
 
 
+void ADsim::updateRuntime(CallListIterator_t exec_it,
+			   unsigned long new_runtime){
+    
+  unsigned long old_exit = exec_it->get_exit();
+  unsigned long old_exit_step = ( old_exit - m_program_start ) / m_step_freq;
+  
+  //Remove the exec from the old step
+  bool found = false;
+  auto &old_step_list = m_step_execs[old_exit_step];
+  for(auto old_step_it = old_step_list.begin(); old_step_it != old_step_list.end(); old_step_it++){
+    if( *old_step_it == exec_it ){
+      old_step_list.erase(old_step_it);
+      found = true;
+      break;
+    }
+  }
+  if(!found) fatal_error("Could not find iterator in old step list!");
+
+  unsigned long new_exit = exec_it->get_entry() + new_runtime;  
+  unsigned long new_exit_step = ( new_exit - m_program_start ) / m_step_freq;
+
+  exec_it->update_exclusive(old_exit);  //hacky way to reset exit time
+  exec_it->update_exit(new_exit);
+
+  m_largest_step = std::max(m_largest_step, new_exit_step);
+
+  //Determine where to insert the entry
+  auto &new_step_list = m_step_execs[new_exit_step];
+  bool inserted = false;
+  
+  if(new_step_list.size() > 1){ //if >2 events the execution might be between existing events
+    unsigned long prev_exit = (*new_step_list.begin())->get_exit();  
+    for(auto new_step_it = std::next(new_step_list.begin(),1); new_step_it != new_step_list.end(); new_step_it++){
+      unsigned long cur_exit = (*new_step_it)->get_exit();
+      if(new_exit >= prev_exit && new_exit <= cur_exit){ //some ambiguity for 0 runtime functions here
+	new_step_list.insert(new_step_it, exec_it);
+	inserted = true;
+	break;
+      }
+      prev_exit = cur_exit;
+    }
+  }
+  
+  if(!inserted)
+    new_step_list.push_back(exec_it);
+}
+
+
+
+
 void ADsim::attachCounter(const std::string &counter_name,
 			  unsigned long value,
 			  long t_delta,

sim/src/ad_params.cpp

@@ -7,6 +7,7 @@ using namespace chimbuko_sim;
 
 void chimbuko_sim::registerFunc(const std::string &func_name, unsigned long mean_runtime, unsigned long std_dev_runtime, int seen_count){ 
   if(adAlgorithmParams().algorithm != "none") fatal_error("This function should only be used if ad_algorithm()==\"none\"");
+  if(seen_count == 0) fatal_error("seen_count must be >0 to generate the fake function statistics");
   unsigned long fidx = registerFunc(func_name);
 
   //Generate fake statistics

sim/src/pserver.cpp

@@ -14,7 +14,9 @@ void pserverSim::writeStreamingOutput() const{
   std::ostringstream fname; fname << "pserver_output_stats_" << iter << ".json";
   ++iter;
 
-  std::ofstream out(fname.str()); out << json_packet.dump();
+  if(json_packet.size() != 0){ //only write if there is anything to write! (it writes "null" to the file otherwise)
+    std::ofstream out(fname.str()); out << json_packet.dump(4);
+  }
 }
 
 pserverSim::pserverSim(): anomaly_stats_payload(&global_func_stats), counter_stats_payload(&global_counter_stats), m_ad_params(nullptr){

src/ad/ADAnomalyProvenance.cpp

@@ -179,7 +179,8 @@ nlohmann::json ADAnomalyProvenance::get_json() const{
 					  {"io_step_tstart", m_io_step_tstart},
 					    {"io_step_tend", m_io_step_tend},
 					      {"outlier_score", m_call.get_outlier_score() },
-						{"hostname", getHostname() }
+						{"outlier_severity", m_call.get_runtime() },
+						  {"hostname", getHostname() }
   };
 }
 

src/ad/ADOutlier.cpp

@@ -148,9 +148,15 @@ double ADOutlierSSTD::computeScore(CallListIterator_t ev, const SstdParam &stats
   double mean = it->second.mean();
   double std_dev = it->second.stddev();
   if(std_dev == 0.) std_dev = 1e-10; //distribution throws an error if std.dev = 0
-  boost::math::normal_distribution<double> dist(mean, std_dev);
-  double cdf_val = boost::math::cdf(dist, runtime); // P( X <= x ) for random variable X
-  double score = std::min(cdf_val, 1-cdf_val); //two-tailed
+
+  //boost::math::normal_distribution<double> dist(mean, std_dev);
+  //double cdf_val = boost::math::cdf(dist, runtime); // P( X <= x ) for random variable X
+  //double score = std::min(cdf_val, 1-cdf_val); //two-tailed
+
+  //Using the CDF gives scores ~0 for basically any global outlier
+  //Instead we will use the difference in runtime compared to the avg in units of the standard deviation
+  double score = fabs( runtime - mean ) / std_dev;
+
   verboseStream << "ADOutlierSSTD::computeScore " << ev->get_funcname() << " runtime " << runtime << " mean " << mean << " std.dev " << std_dev << " -> score " << score << std::endl;
   return score;
 }