Updated NWChem+Chimbuko Dockerfile and scripts

	Configuration uses HBOS by default and includes monitoring plugin
	Run scripts now supports BP4 tracing
	Disable NWChem ADIOS2 output in config

Author: giltirn

docker/ubuntu18.04/openmpi4.0.4/Dockerfile.chimbuko.nwchem

@@ -19,9 +19,11 @@ ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/tau2/x86_64/lib
 ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/adios2/lib
 ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/SZ/lib
 
+RUN apt-get update && apt-get install -y gdb emacs-nox vim
+
 WORKDIR /
 
-RUN echo "INVALIDATE_2" > /dev/null
+RUN echo "INVALIDATE_3" > /dev/null
 COPY run_nwchem_chimbuko.sh /
 COPY sos_filter.txt /
 COPY chimbuko_config.templ /

docker/ubuntu18.04/openmpi4.0.4/chimbuko_config.templ

@@ -26,15 +26,18 @@ chimbuko_services="infer" #The location of the Chimbuko service script. If set t
 use_provdb=1 #enable or disable the provDB. If disabled the provenance data will be written as JSON ASCII into the ${provdb_writedir} set below
 provdb_extra_args="" #any extra command line arguments to pass
 provdb_nshards=4  #number of database shards
+provdb_ninstances=1 #number of database server instances. Shards are distributed over instances
 provdb_engine="ofi+tcp;ofi_rxm"  #the OFI libfabric provider used for the Mochi stack
 provdb_port=5000 #the port of the provenance database
-provdb_nthreads=4 #number of worker threads; should be >= the number of shards
 provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default
 provdb_commit_freq=10000   #frequency ms at which the provenance database is committed to disk. If set to 0 it will commit only at the end
 
 #With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
 provdb_domain=mlx5_0 #only needed for verbs provider   <------------ ***SET ME (if using verbs)***
 
+export FI_UNIVERSE_SIZE=1600  # Defines the expected number of provenance DB clients per instance <------------- *** SET ME (should be larger than the number of clients/instance)
+export FI_MR_CACHE_MAX_COUNT=0   # disable MR cache in libfabric; still problematic as of libfabric 1.10.1
+export FI_OFI_RXM_USE_SRX=1 # use shared recv context in RXM; should improve scalability
 
 ####################################
 #Options for the parameter server
@@ -48,7 +51,7 @@ pserver_nt=2 #number of worker threads
 ####################################
 ad_extra_args="-perf_outputpath chimbuko/logs -perf_step 1 -interval_msec <AD_INTERVAL>" #any extra command line arguments to pass. Note: chimbuko/logs is automatically created by services script
 ad_win_size=<AD_WIN_SIZE>  #number of events around an anomaly to store; provDB entry size is proportional to this so keep it small!
-ad_alg="sstd" #the anomaly detection algorithm. Valid values are "hbos" and "sstd"
+ad_alg="hbos" #the anomaly detection algorithm. Valid values are "hbos" and "sstd"
 ad_outlier_hbos_threshold=0.99  #the percentile of events outside of which are considered anomalies by the HBOS algorithm
 ad_outlier_sstd_sigma=<AD_SIGMA> #number of standard deviations that defines an outlier in the SSTD algorithm
 ####################################
@@ -64,13 +67,15 @@ export TAU_ADIOS2_PERIOD=1000000  #period in us between ADIOS2 io steps
 export TAU_THREAD_PER_GPU_STREAM=1  #force GPU streams to appear as different TAU virtual threads
 export TAU_THROTTLE=0  #enable/disable throttling of short-running functions
 
-export TAU_MAKEFILE=/opt/tau2/x86_64/lib/Makefile.tau-papi-mpi-pthread-pdt-adios2  #The TAU makefile to use <------------ ***SET ME***
+export TAU_MAKEFILE=/opt/tau2/x86_64/lib/Makefile.tau-papi-mpi-pthread-pdt-adios2  #The TAU makefile to use. If using a TAU installation built by Spack, this variable is already set in the environment and can be commented out here <------------ ***SET ME***
+
+tau_monitoring_conf="default" #Provide a configuration file for the TAU monitoring plugin. It will be copied to the work directory as "tau_monitoring.json" (unless it is already there!). If set to default, Chimbuko will generate one automatically
 
 #Note: the following 2 variables are not used by the service script but are included here for use from the user's run script allowing the application to be launched with either "${TAU_EXEC} <app>" or "${TAU_PYTHON} <app>"
 #Note: the "binding" -T ... is used by Tau to find the appropriate configuration. It can typically be inferred from the name of the Makefile. If using a non-MPI job the 'mpi' should be changed to 'serial' and a non-MPI build of
 #      ADIOS2/TAU must exist
 #Suggestion: It is useful to test the command without Chimbuko first to ensure TAU picks up the correct binding; this can be done by 'export TAU_ADIOS2_ENGINE=BPFile' and then running the application with Tau but without Chimbuko.
-TAU_EXEC="tau_exec -T papi,mpi,pthread,pdt,adios2 -adios2_trace"   #how to execute tau_exec; the -T arguments should mirror the makefile name  <------------ ***SET ME***
+TAU_EXEC="tau_exec -T papi,mpi,pthread,pdt,adios2 -adios2_trace -monitoring"   #how to execute tau_exec; the -T arguments should mirror the makefile name  <------------ ***SET ME***
 TAU_PYTHON="tau_python -T papi,mpi,pthread,pdt,adios2 -tau-python-interpreter=python3 -adios2_trace  -tau-python-args=-u"  #how to execute tau_python. Note that passing -u to python forces it to not buffer stdout so we can pipe it
                                                                                                                            #to tee in realtime <--- SET ME (if !python3)
 
@@ -98,6 +103,11 @@ TAU_ADIOS2_FILE_PREFIX=tau-metrics  #the prefix of tau adios2 files; full filena
 export TAU_ADIOS2_FILENAME="${TAU_ADIOS2_PATH}/${TAU_ADIOS2_FILE_PREFIX}"
 
 if [[ ${backend_root} == "infer" ]]; then
+    if [[ $(which provdb_admin) == "" ]]; then
+	echo "When inferring the backend root directory, could not find provdb_admin in PATH. Please add your Chimbuko bin directory to PATH"
+	exit 1
+    fi
+    
     backend_root=$( readlink -f $(which provdb_admin | sed 's/provdb_admin//')/../ )
 fi
 

docker/ubuntu18.04/openmpi4.0.4/run_nwchem_chimbuko.sh

@@ -83,7 +83,7 @@ sed -i 's/coord 0/coord 1/' ethanol_md.nw
 sed -i 's/scoor 0/scoor 1/' ethanol_md.nw
 sed -i 's/step 0.001/step 0.001/' ethanol_md.nw
 sed -i '21s|set|#set|' ethanol_md.nw
-sed -i '22s|#set|set|' ethanol_md.nw
+sed -i '17s|#set|set|' ethanol_md.nw
 sed -i "s|data 1000|data ${DATA_STEPS}|" ethanol_md.nw
 
 if (( 1 )); then
@@ -94,7 +94,7 @@ if (( 1 )); then
     ad_cmd=$(cat chimbuko/vars/chimbuko_ad_cmdline.var)
 fi
 
-if [ "$ADIOS_MODE" == "SST" ]
+if [[ "$ADIOS_MODE" == "SST" || "$ADIOS_MODE" == "BP4" ]]
 then
     echo "Launch Application with anomaly detectors"
     eval "mpirun --allow-run-as-root -n ${NMPIS} ${ad_cmd} &"