In ADEvent, detection of multiple correlation IDs for GPU events is no longer a fatal error but instead a recoverable error. Modified unit test accordingly.

ADAnomalyProvenance
	Detection of multiple correlation IDs for GPU events is no longer a fatal error; instead the provenance data is provided with an error message to record
	Missing correlation IDs or cases where the parent event has somehow become deleted now also record error messages to the provenance data
	Added unit tests of the above

Author: giltirn

src/ad/ADAnomalyProvenance.cpp

@@ -49,46 +49,56 @@ void ADAnomalyProvenance::getGPUeventInfo(const ExecData_t &call, const ADEvent
       //Note a GPU event can only be partnered to one CPU event but a CPU event can be partnered to multiple GPU events
       if(call.n_GPU_correlationID_partner() != 1){
 	std::stringstream ss; ss << "GPU event has multiple correlation ID partners?? Event details:" << std::endl << call.get_json(false,true).dump() << std::endl;
-	fatal_error(ss.str());
-      }
-
-      verboseStream << "Call has a GPU correlation ID partner: " <<  call.get_GPU_correlationID_partner(0).toString() << std::endl;
-
-      const eventID &gpu_event_parent = call.get_GPU_correlationID_partner(0);
-      m_gpu_event_parent_info["event_id"] = gpu_event_parent.toString();
-
-      //Get the parent event
-      CallListIterator_t pit;
-      bool got_parent = true;
-      try{
-	pit = event_man.getCallData(gpu_event_parent);
-      }catch(const std::exception &e){
-	recoverable_error("Could not find GPU parent " + gpu_event_parent.toString() + " in call list due to : " + e.what());
-	got_parent = false;
-      }
-
-      if(got_parent){
-	m_gpu_event_parent_info["tid"] = pit->get_tid();
-
-	//Generate the parent stack
-	nlohmann::json gpu_event_parent_stack = nlohmann::json::array();
-	gpu_event_parent_stack.push_back(getCallStackEntry(*pit));
-
-	eventID parent = pit->get_parent();
-	while(parent != eventID::root()){
-	  CallListIterator_t call_it;
-	  try{
-	    call_it = event_man.getCallData(parent);
-	  }catch(const std::exception &e){
-	    recoverable_error("Could not find GPU stack event parent " + parent.toString() + " in call list due to : " + e.what());
-	    break;
+	recoverable_error(ss.str());
+
+	m_gpu_event_parent_info = "Chimbuko error: Multiple host parent event correlation IDs found, likely due to trace corruption";
+      }else{
+	verboseStream << "Call has a GPU correlation ID partner: " <<  call.get_GPU_correlationID_partner(0).toString() << std::endl;
+
+	const eventID &gpu_event_parent = call.get_GPU_correlationID_partner(0);
+	m_gpu_event_parent_info["event_id"] = gpu_event_parent.toString();
+
+	//Get the parent event
+	CallListIterator_t pit;
+	bool got_parent = true;
+	try{
+	  pit = event_man.getCallData(gpu_event_parent);
+	}catch(const std::exception &e){
+	  recoverable_error("Could not find GPU parent " + gpu_event_parent.toString() + " in call list due to : " + e.what());
+	  got_parent = false;
+	}
+
+	if(got_parent){
+	  m_gpu_event_parent_info["tid"] = pit->get_tid();
+
+	  //Generate the parent stack
+	  nlohmann::json gpu_event_parent_stack = nlohmann::json::array();
+	  gpu_event_parent_stack.push_back(getCallStackEntry(*pit));
+
+	  eventID parent = pit->get_parent();
+	  while(parent != eventID::root()){
+	    CallListIterator_t call_it;
+	    try{
+	      call_it = event_man.getCallData(parent);
+	    }catch(const std::exception &e){
+	      recoverable_error("Could not find GPU stack event parent " + parent.toString() + " in call list due to : " + e.what());
+	      break;
+	    }
+	    gpu_event_parent_stack.push_back(getCallStackEntry(*call_it));
+	    parent = call_it->get_parent();
 	  }
-	  gpu_event_parent_stack.push_back(getCallStackEntry(*call_it));
-	  parent = call_it->get_parent();
+	  m_gpu_event_parent_info["call_stack"] = std::move(gpu_event_parent_stack);
+	}else{
+	  //Could not get parent event
+	  m_gpu_event_parent_info = "Chimbuko error: Host parent event could not be reached";
 	}
-	m_gpu_event_parent_info["call_stack"] = std::move(gpu_event_parent_stack);
-      }
-    }//have correlation ID partner
+
+      }//have *one* correlation ID partner
+    }else{ 
+      //No correlation ID recorded
+      m_gpu_event_parent_info = "Chimbuko error: Correlation ID of host parent event was not recorded";
+    }
+
   }//m_is_gpu_event
 }
 

src/ad/ADEvent.cpp

@@ -119,7 +119,7 @@ void ADEvent::checkAndMatchCorrelationID(CallListIterator_t it){
        << it->get_json(false, true).dump() << std::endl
        << "GPU details: " << std::endl
        << m_gpu_thread_Map->find(it->get_tid())->second.get_json().dump() << std::endl;
-    fatal_error(ss.str());
+    recoverable_error(ss.str());
   }
 }
 

test/unit_tests/ad/ADAnomalyProvenance.cpp

@@ -249,6 +249,162 @@ TEST(TestADAnomalyProvenance, detectsGPUevents){
 
 
 
+TEST(TestADAnomalyProvenance, gracefullyFailsIfCorrelationIDissues){
+  int gpu_thr = 9;
+  int corrid_cid = 22; //counter index!
+  
+  ADEvent event_man;
+
+  //Populate all the other stuff required to generate anomaly data
+  RunStats stats;
+  for(int i=0;i<50;i++)
+    stats.push(double(i));
+
+  SstdParam param;
+  param[44] = stats;
+  param[55] = stats;
+  param[11] = stats;
+
+  ADCounter counter;
+
+  ADMetadataParser metadata;
+  std::vector<MetaData_t> mdata = {
+    MetaData_t(0,0, gpu_thr, "CUDA Context", "8"),
+    MetaData_t(0,0, gpu_thr, "CUDA Stream", "1"),
+    MetaData_t(0,0, gpu_thr, "CUDA Device", "7"),
+    MetaData_t(0,0, gpu_thr, "GPU[7] Device Name", "Fake GPU")
+  };
+  metadata.addData(mdata);
+  
+  {
+    std::cout <<  "Testing failure due to missing correlation ID" << std::endl;
+
+    //Have a host correlation ID but not a device one
+    int corridx1 = 1234;
+    ExecData_t exec_gpu = createFuncExecData_t(0,1, gpu_thr, 55, "thegpufunction", 1000, 100); //on gpu
+    
+    ExecData_t exec_cpu = createFuncExecData_t(0,1, 0, 44, "thecpufunction", 1000, 100); //not on gpu
+    exec_cpu.add_counter(createCounterData_t(0,1, 0, corrid_cid,  corridx1, 1000, "Correlation ID"));
+    
+    CallListIterator_t exec_cpu_it = event_man.addCall(exec_cpu);    
+    CallListIterator_t exec_gpu_it = event_man.addCall(exec_gpu);
+    
+    ADAnomalyProvenance prov_gpu(*exec_gpu_it,
+				 event_man,
+				 param,
+				 counter, metadata, 0,
+				 11,900,1200);
+    {
+      nlohmann::json output = prov_gpu.get_json();
+      std::cout << "For GPU event, got: " << output.dump() << std::endl;
+    
+      EXPECT_EQ(output["is_gpu_event"], true);
+      EXPECT_EQ(output["gpu_location"]["context"], 8);
+      EXPECT_EQ(output["gpu_location"]["device"], 7);
+      EXPECT_EQ(output["gpu_location"]["stream"], 1);
+
+      std::string got = output["gpu_parent"];
+      std::string expect = "Chimbuko error: Correlation ID of host parent event was not recorded";
+      std::cout << got << std::endl;
+
+      EXPECT_EQ(got, expect);
+    }
+  }
+
+  //Failure due to multiple correlation IDs
+  {
+    std::cout <<  "Testing failure due to multiple correlation IDs" << std::endl;
+    int corridx2 = 2222, corridx3 = 3333;
+
+    ExecData_t exec_gpu = createFuncExecData_t(0,1, gpu_thr, 55, "thegpufunction", 1000, 100); //on gpu
+    exec_gpu.add_counter(createCounterData_t(0,1, gpu_thr, corrid_cid,  corridx2, 1000, "Correlation ID")); //this one has 2 correlation IDs
+    exec_gpu.add_counter(createCounterData_t(0,1, gpu_thr, corrid_cid,  corridx3, 1000, "Correlation ID"));
+    
+    ExecData_t exec_cpu = createFuncExecData_t(0,1, 0, 44, "thecpufunction", 1000, 100); //not on gpu
+    exec_cpu.add_counter(createCounterData_t(0,1, 0, corrid_cid,  corridx2, 1000, "Correlation ID"));
+
+    ExecData_t exec_cpu2 = createFuncExecData_t(0,1, 0, 66, "theothercpufunction", 1000, 100); //not on gpu
+    exec_cpu2.add_counter(createCounterData_t(0,1, 0, corrid_cid,  corridx3, 1000, "Correlation ID"));
+
+    CallListIterator_t exec_cpu_it = event_man.addCall(exec_cpu);
+    CallListIterator_t exec_cpu2_it = event_man.addCall(exec_cpu2);
+    
+    CallListIterator_t exec_gpu_it = event_man.addCall(exec_gpu);
+
+    ADAnomalyProvenance prov_gpu(*exec_gpu_it,
+				 event_man,
+				 param,
+				 counter, metadata, 0,
+				 11,900,1200);
+    {
+      nlohmann::json output = prov_gpu.get_json();
+      std::cout << "For GPU event, got: " << output.dump() << std::endl;
+
+      EXPECT_EQ(output["is_gpu_event"], true);
+      EXPECT_EQ(output["gpu_location"]["context"], 8);
+      EXPECT_EQ(output["gpu_location"]["device"], 7);
+      EXPECT_EQ(output["gpu_location"]["stream"], 1);
+
+      std::string got = output["gpu_parent"];
+      std::string expect = "Chimbuko error: Multiple host parent event correlation IDs found, likely due to trace corruption";
+      std::cout << got << std::endl;
+
+      EXPECT_EQ(got, expect);
+    }
+  }
+
+  {
+    std::cout <<  "Testing failure due to missing parent event" << std::endl;
+
+    //Have a host correlation ID but not a device one
+    int corridx4 = 4444;
+    ExecData_t exec_gpu = createFuncExecData_t(0,1, gpu_thr, 55, "thegpufunction", 1000, 100); //on gpu
+    exec_gpu.add_counter(createCounterData_t(0,1, gpu_thr, corrid_cid,  corridx4, 1000, "Correlation ID")); //this one has 2 correlation IDs
+    
+    ExecData_t exec_cpu = createFuncExecData_t(0,1, 0, 44, "thecpufunction", 1000, 100); //not on gpu
+    exec_cpu.add_counter(createCounterData_t(0,1, 0, corrid_cid,  corridx4, 1000, "Correlation ID"));
+    
+    CallListIterator_t exec_cpu_it = event_man.addCall(exec_cpu);
+
+    //Force the trimming out of the cpu event
+    exec_cpu_it->can_delete(true);
+    delete event_man.trimCallList();
+
+    CallListIterator_t exec_gpu_it = event_man.addCall(exec_gpu);
+    
+    ADAnomalyProvenance prov_gpu(*exec_gpu_it,
+				 event_man,
+				 param,
+				 counter, metadata, 0,
+				 11,900,1200);
+    {
+      nlohmann::json output = prov_gpu.get_json();
+      std::cout << "For GPU event, got: " << output.dump() << std::endl;
+    
+      EXPECT_EQ(output["is_gpu_event"], true);
+      EXPECT_EQ(output["gpu_location"]["context"], 8);
+      EXPECT_EQ(output["gpu_location"]["device"], 7);
+      EXPECT_EQ(output["gpu_location"]["stream"], 1);
+
+      std::string got = output["gpu_parent"];
+      std::string expect = "Chimbuko error: Host parent event could not be reached";
+      std::cout << got << std::endl;
+
+      EXPECT_EQ(got, expect);
+    }
+  }
+
+
+  
+
+
+}
+
+
+
+
+
+
 TEST(TestADAnomalyProvenance, extractsExecWindow){
   ExecData_t exec0 = createFuncExecData_t(1,2,3, 33, "theonebefore", 900, 0); //not yet completed
   ExecData_t exec1 = createFuncExecData_t(1,2,3, 55, "theparent", 1000, 100);

test/unit_tests/ad/ADEvent.cpp

@@ -1,5 +1,6 @@
 #include<chimbuko/ad/ADEvent.hpp>
 #include<chimbuko/util/map.hpp>
+#include<chimbuko/util/error.hpp>
 #include "gtest/gtest.h"
 #include "../unit_test_common.hpp"
 
@@ -662,25 +663,31 @@ TEST(ADEventTest, DetectsCorrelationIDerrors){
   event_man.addCounter(gpu_corrid1);
   event_man.addCounter(gpu_corrid2);
 
-  //GPU events not allowed to have multiple correlation IDs
-  bool got_error = false;
-  try{
+  //GPU events not allowed to have multiple correlation IDs (non-fatal)
+  {
+    std::stringstream err_str;
+    Error().setStream(&err_str);
+
     event_man.addFunc(gpu_exit);
-  }catch(const std::exception &err){
-    std::cout << "Caught intentional error: " << err.what() << std::endl;
-    got_error = true;
+
+    std::string got = err_str.str();
+    size_t loc = got.find("Encountered a GPU kernel execution with multiple correlation IDs!");
+    std::cout << "Got intentional error: " << got << std::endl;
+    EXPECT_NE(loc, std::string::npos);
   }
-  EXPECT_EQ(got_error, true);
 
   //CPU events are allowed to have multiple correlation IDs
-  event_man.addCounter(cpu_corrid2);
+  {
+    event_man.addCounter(cpu_corrid2);
+    
+    std::stringstream err_str;
+    Error().setStream(&err_str);
 
-  got_error = false;
-  try{
     event_man.addFunc(cpu_exit);
-  }catch(const std::exception &err){
-    std::cout << "Caught *un*intentional error: " << err.what() << std::endl;
-    got_error = true;
+
+    std::string got = err_str.str();
+    size_t loc = got.find("Encountered a GPU kernel execution with multiple correlation IDs!");
+    std::cout << "This should be empty: " << got << std::endl;
+    EXPECT_EQ(loc, std::string::npos);
   }
-  EXPECT_EQ(got_error, false);
 }