Merge branch 'ckelly_develop' of https://github.com/CODARcode/PerformanceAnalysis into ckelly_develop

Author: giltirn

sphinx/source/io_schema/pserver_schema.rst

@@ -75,7 +75,7 @@ The schema for the **'anomaly_stats'** object is as follows:
 |                   ...
 |                ],
 |           **'key'**: *A string label of the form "$PROGRAM ID:$RANK" (eg "0:12")*,
-|           **'stats'**:   *Statistics on anomalies on this process/rank over the steps broken down above (object)*
+|           **'stats'**:   *Statistics on anomalies on this process/rank over all steps to date
 |                {
 |	           **'accumulate'**: *Total anomalies*,
 |                  **'count'**: *Number of io steps included in statistics*,

src/pserver/global_anomaly_stats.cpp

@@ -72,22 +72,34 @@ nlohmann::json GlobalAnomalyStats::collect_stat_data(){
   //AnomalyStat contains statistics on the number of anomalies found per io step and also a set of AnomalyData objects
   //that have been collected from that rank since the last flush
   for(auto & pp : m_anomaly_stats){
-    int pid = pp.first;
+    int pid = pp.first; //pid
     for(auto & rp: pp.second){
-      unsigned long rid = rp.first;
+      unsigned long rid = rp.first; //rank
 
       auto stats = rp.second.get(); //returns a std::pair<RunStats, std::list<AnomalyData>*>,  and flushes the state of pair.second. 
       //We now own the std::list<AnomalyData>* pointer and have to delete it
 
       if(stats.second){
-	if(stats.second->size()){
+	//Decide whether to include the data for this pid/rid
+	//Do this only if any anomalies were seen since the last call
+	bool include = false;
+	for(const AnomalyData &adata: *stats.second){
+	  if(adata.get_n_anomalies() > 0){
+	    include = true;
+	    break;
+	  }
+	}
+
+	if(include){
 	  nlohmann::json object;
 	  object["key"] = stringize("%d:%d", pid,rid);
-	  object["stats"] = stats.first.get_json();
+	  object["stats"] = stats.first.get_json(); //statistics on anomalies to date for this pid/rid
 
 	  object["data"] = nlohmann::json::array();
-	  for (auto const &adata: *stats.second){
-	    object["data"].push_back(adata.get_json());
+	  for (const AnomalyData &adata: *stats.second){
+	    //Don't include data for which there are no anomalies
+	    if(adata.get_n_anomalies()>0)
+	      object["data"].push_back(adata.get_json());
 	  }
 	  jsonObjects.push_back(object);
 	}

test/unit_tests/pserver/global_anomaly_stats.cpp

@@ -107,5 +107,44 @@ TEST(TestGlobalAnomalyStats, UpdateAnomalyStat){
   //Check data was flushed
   adata = astats.get_data();
   EXPECT_EQ(adata->size(), 0);
+
+  //Check that collect_stat_data only includes data if the number of anomalies seen since last collect > 0
+  int step3= 2;
+  int min_ts3 = 2000;
+  int max_ts3 = 3000;
+  int n_anomalies3 = 0;
+
+  AnomalyData d3(app, rank, step3, min_ts3, max_ts3, n_anomalies3);
+  
+  stats.update_anomaly_stat(d3);
+  
+  astats_j = stats.collect_stat_data();
+  EXPECT_EQ(astats_j.is_array(), true );
+  EXPECT_EQ(astats_j.size(), 0 );
+  
+  //Check that if there are multiple steps, some of which have anomalies, that the data array only contains the data with nanom > 0
+
+  int step4= 3;
+  int min_ts4 = 3000;
+  int max_ts4 = 4000;
+  int n_anomalies4 = 0;
+
+  AnomalyData d4(app, rank, step4, min_ts4, max_ts4, n_anomalies4);
+
+  int step5= 4;
+  int min_ts5 = 4000;
+  int max_ts5 = 5000;
+  int n_anomalies5 = 10; 
+
+  AnomalyData d5(app, rank, step5, min_ts5, max_ts5, n_anomalies5);
+
+  stats.update_anomaly_stat(d4);
+  stats.update_anomaly_stat(d5);
+
+  astats_j = stats.collect_stat_data();
+  EXPECT_EQ(astats_j.is_array(), true );
+  EXPECT_EQ(astats_j.size(), 1 ); //1 rank
+  EXPECT_EQ(astats_j[0]["data"].size(), 1); //only d5
+  EXPECT_EQ(astats_j[0]["data"][0]["step"], step5);
 }