It seems that with CUDA 12.4 and the latest tau that GPU correlation IDs now appear associated with the kernel exit rather than entry. To combat this, I disabled the special-case ordering for correlation ID event parsing. Will need to revisit this issue for other versions / GPU APIs

Updated default config script for benchmark_suite/cupti_gpu_kernel_outlier

Author: giltirn

benchmark_suite/cupti_gpu_kernel_outlier/chimbuko_config.sh

@@ -3,7 +3,7 @@
 
 #IMPORTANT NOTE: Variables that cannot be left as default are marked as <------------ ***SET ME***
 
-service_node_iface=eth0 #network interface upon which communication to the service node is performed <------------ ***SET ME***
+module="performance_analysis"
 
 ####################################
 #Options for visualization module
@@ -27,13 +27,31 @@ use_provdb=1 #enable or disable the provDB. If disabled the provenance data will
 provdb_extra_args="" #any extra command line arguments to pass
 provdb_nshards=4  #number of database shards
 provdb_ninstances=1 #number of database server instances. Shards are distributed over instances
-provdb_engine="ofi+tcp;ofi_rxm"  #the OFI libfabric provider used for the Mochi stack
+provdb_engine="sockets"  #the OFI libfabric provider used for the Mochi stack
 provdb_port=5000 #the port of the provenance database
 provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default
 provdb_commit_freq=10000   #frequency ms at which the provenance database is committed to disk. If set to 0 it will commit only at the end
 
-#With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
-provdb_domain=mlx5_0 #only needed for verbs provider   <------------ ***SET ME (if using verbs)***
+#provdb_interface : network interface upon which communication to the provdb is performed. <------------ ***SET ME***
+# This variable has several options:
+#  auto                             - let Mercury automatically choose an interface for all instances
+#  <iface>                          - a single interface used for all instances
+#  <iface1>:<iface2>:<iface3> ....  - a colon-separated list of interfaces, one per instance
+# Obtain a list of interfaces from, e.g. "ip link show" (cf https://www.cyberciti.biz/faq/linux-list-network-interfaces-names-command/).
+provdb_interface=auto
+
+#provdb_domain : With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
+#  If left blank it will be chosen automatically. <------------ ***SET ME (if using verbs)***
+provdb_domain=
+
+#provdb_numa_bind : specify NUMA domain binding for the provdb instances (requires numactl)
+# This variable has several options:
+# <blank>  - if left blank, no binding will be performed
+# <index>  - a single NUMA domain for all instances
+# <idx1>:<idx2>:<idx3> ...  - a colon-separated list of NUMA domains, one per instance
+provdb_numa_bind=
+
+commit_extra_args="" #extra arguments for the committer
 
 export FI_UNIVERSE_SIZE=1600  # Defines the expected number of provenance DB clients per instance <------------- *** SET ME (should be larger than the number of clients/instance)
 export FI_MR_CACHE_MAX_COUNT=0   # disable MR cache in libfabric; still problematic as of libfabric 1.10.1
@@ -44,14 +62,17 @@ export FI_OFI_RXM_USE_SRX=1 # use shared recv context in RXM; should improve sca
 ####################################
 use_pserver=1 #enable or disable the pserver
 pserver_extra_args="" #any extra command line arguments to pass
+pserver_interface=eth0 #network interface upon which communication to the pserver is performed. Obtain from, e.g. "ip link show" (cf https://www.cyberciti.biz/faq/linux-list-network-interfaces-names-command/). <------------ ***SET ME***
 pserver_port=5559  #port for parameter server
 pserver_nt=2 #number of worker threads
+pserver_numa_bind= #specify NUMA domain binding for the pserver (requires numactl). If left blank, no binding will be performed
+
 ####################################
 #Options for the AD module
 ####################################
 ad_extra_args="-perf_outputpath chimbuko/logs -perf_step 1" #any extra command line arguments to pass. Note: chimbuko/logs is automatically created by services script
 ad_win_size=5  #number of events around an anomaly to store; provDB entry size is proportional to this so keep it small!
-ad_alg="sstd" #the anomaly detection algorithm. Valid values are "hbos" and "sstd"
+ad_alg="hbos" #the anomaly detection algorithm. Valid values are "hbos" and "sstd"
 ad_outlier_hbos_threshold=0.99  #the percentile of events outside of which are considered anomalies by the HBOS algorithm
 ad_outlier_sstd_sigma=12 #number of standard deviations that defines an outlier in the SSTD algorithm
 ####################################
@@ -60,20 +81,22 @@ ad_outlier_sstd_sigma=12 #number of standard deviations that defines an outlier
 #      command to launch the AD (output to chimbuko/vars/chimbuko_ad_cmdline.var); they can be overridden by the run script if desired providing the appropriate modifications
 #      are made to the AD launch command. The remainder of the variables are used only by TAU and can be freely overridden.
 ####################################
-export TAU_ADIOS2_ENGINE=SST    #online communication engine (alternative BP4 although this goes through the disk system and may be slower unless the BPfiles are stored on a burst disk)
+export TAU_ADIOS2_ENGINE=BP4   #online communication engine (alternative BP4 although this goes through the disk system and may be slower unless the BPfiles are stored on a burst disk)
 export TAU_ADIOS2_ONE_FILE=FALSE  #a different connection file for each rank
 export TAU_ADIOS2_PERIODIC=1   #enable/disable ADIOS2 periodic output
 export TAU_ADIOS2_PERIOD=1000000  #period in us between ADIOS2 io steps
 export TAU_THREAD_PER_GPU_STREAM=1  #force GPU streams to appear as different TAU virtual threads
 export TAU_THROTTLE=0  #enable/disable throttling of short-running functions
 
-export TAU_MAKEFILE=/opt/tau2/x86_64/lib/Makefile.tau-papi-mpi-pthread-python-cupti-pdt-adios2  #The TAU makefile to use <------------ ***SET ME***
+#export TAU_MAKEFILE=/opt/tau2/x86_64/lib/Makefile.tau-papi-mpi-pthread-python-cupti-pdt-adios2  #The TAU makefile to use <------------ ***SET ME***
+
+tau_monitoring_conf="default" #Provide a configuration file for the TAU monitoring plugin. It will be copied to the work directory as "tau_monitoring.json" (unless it is already there!). If set to default, Chimbuko will generate one automatically
 
 #Note: the following 2 variables are not used by the service script but are included here for use from the user's run script allowing the application to be launched with either "${TAU_EXEC} <app>" or "${TAU_PYTHON} <app>"
 #Note: the "binding" -T ... is used by Tau to find the appropriate configuration. It can typically be inferred from the name of the Makefile. If using a non-MPI job the 'mpi' should be changed to 'serial' and a non-MPI build of
 #      ADIOS2/TAU must exist
 #Suggestion: It is useful to test the command without Chimbuko first to ensure TAU picks up the correct binding; this can be done by 'export TAU_ADIOS2_ENGINE=BPFile' and then running the application with Tau but without Chimbuko.
-TAU_EXEC="tau_exec -T papi,mpi,pthread,python,cupti,pdt,adios2 -adios2_trace -cupti -um"  #how to execute tau_exec; the -T arguments should mirror the makefile name  <------------ ***SET ME***
+TAU_EXEC="tau_exec -T papi,mpi,pthread,cupti,pdt,adios2 -cupti -adios2_trace -monitoring"  #how to execute tau_exec; the -T arguments should mirror the makefile name  <------------ ***SET ME***   #-um
 TAU_PYTHON="tau_python -T papi,mpi,pthread,python,cupti,pdt,adios2 -tau-python-interpreter=python3 -adios2_trace  -tau-python-args=-u"  #how to execute tau_python. Note that passing -u to python forces it to not buffer stdout so we can pipe it
                                                                                                                            #to tee in realtime <--- SET ME (if !python3)
 

src/modules/performance_analysis/ad/ADParser.cpp

@@ -707,7 +707,10 @@ std::vector<Event_t> ADParser::getEvents() const{
       }else if(func_event_type == EXIT && counter_is_correlation_id){
 	//Correlation IDs are associated only with function ENTRY events; if an EXIT event coincides with a CorrelationID the EXIT takes priority over counter events
 	//Functions are still greedy with comm events however
-	SET_PRIO(COMM, FUNC, COUNTER);
+	//SET_PRIO(COMM, FUNC, COUNTER);
+
+	//EDIT 4/10/25: This no longer appears to be the case; at least for my recent testing on CUDA 12.4 and recent TAU, the correlation ID is associated with the function exit. To check, run sstSinker on a GPU program and look where the kernel-side correlation ID appears relative to the stimestamps. Will need to check this on other versions / ROCM.
+	SET_PRIO(COMM, COUNTER, FUNC);
       }else{
 	//Otherwise funcData takes lowest priority so comm and counter events are included in the function execution
 	SET_PRIO(COMM, COUNTER, FUNC);