Added ability for pserver to write provenance data as plain text using -prov_outputpath cmdline option

Services script now allows for disabling of pserver and provDB. If provDB is not used the pserver and AD will be automatically set to write provenance as plain text
Services script now waits for the provDB to exit before finishing up

Author: giltirn

app/pserver.cpp

@@ -54,7 +54,9 @@ struct pserverArgs{
   std::string provdb_addr;
 #endif
 
-  pserverArgs(): ad("hbos"), nt(-1), logdir("."), ws_addr(""), load_params_set(false), save_params_set(false), freeze_params(false), stat_send_freq(1000), stat_outputdir(""), port(5559)
+  std::string prov_outputpath;
+
+  pserverArgs(): ad("hbos"), nt(-1), logdir("."), ws_addr(""), load_params_set(false), save_params_set(false), freeze_params(false), stat_send_freq(1000), stat_outputdir(""), port(5559), prov_outputpath("")
 #ifdef _USE_ZMQNET
 	       , max_pollcyc_msg(10), zmq_io_thr(1), autoshutdown(true)
 #endif
@@ -85,6 +87,8 @@ struct pserverArgs{
 #ifdef ENABLE_PROVDB
       addOptionalCommandLineArg(p, provdb_addr, "Address of the provenance database. If empty (default) the global function and counter statistics will not be send to the provenance DB.\nHas format \"ofi+tcp;ofi_rxm://${IP_ADDR}:${PORT}\". Should also accept \"tcp://${IP_ADDR}:${PORT}\"");
 #endif
+      addOptionalCommandLineArg(p, prov_outputpath, "Output global provenance data to this directory. Can be used in place of or in conjunction with the provenance database. An empty string \"\" (default) disables this output");
+
 
       init = true;
     }
@@ -210,13 +214,25 @@ int main (int argc, char ** argv){
     stat_sender.stop_stat_sender(1000);
 
 #ifdef ENABLE_PROVDB
-    //Send final statistics to the provenance database
-    if(provdb_client.isConnected()){
-      progressStream << "Pserver: sending final statistics to provDB" << std::endl;
-      provdb_client.sendMultipleData(global_func_stats.collect_func_data(), GlobalProvenanceDataType::FunctionStats);
-      provdb_client.sendMultipleData(global_counter_stats.get_json_state(), GlobalProvenanceDataType::CounterStats);
-      progressStream << "Pserver: disconnecting from provDB" << std::endl;
-      provdb_client.disconnect();
+    //Send final statistics to the provenance database and/or disk
+    if(provdb_client.isConnected() || args.prov_outputpath.size() > 0){
+      nlohmann::json global_func_stats_j = global_func_stats.collect_func_data();
+      nlohmann::json global_counter_stats_j = global_counter_stats.get_json_state();
+
+      if(provdb_client.isConnected()){
+	progressStream << "Pserver: sending final statistics to provDB" << std::endl;
+	provdb_client.sendMultipleData(global_func_stats_j, GlobalProvenanceDataType::FunctionStats);
+	provdb_client.sendMultipleData(global_counter_stats_j, GlobalProvenanceDataType::CounterStats);
+	progressStream << "Pserver: disconnecting from provDB" << std::endl;
+	provdb_client.disconnect();
+      }
+      if(args.prov_outputpath.size() > 0){
+	progressStream << "Pserver: writing final statistics to disk at path " << args.prov_outputpath << std::endl;
+	std::ofstream gf(args.prov_outputpath + "/global_func_stats.json");
+	std::ofstream gc(args.prov_outputpath + "/global_counter_stats.json");
+	gf << global_func_stats_j.dump();
+	gc << global_counter_stats_j.dump();
+      }
     }
 #endif
 

scripts/launch/chimbuko_config.sh

@@ -22,6 +22,7 @@ backend_root="infer"  #The root install directory of the PerformanceAnalysis lib
 ####################################
 #Options for the provenance database
 ####################################
+use_provdb=1 #enable or disable the provDB. If disabled the provenance data will be written as JSON ASCII into the ${provdb_writedir} set below
 provdb_extra_args="" #any extra command line arguments to pass
 provdb_nshards=4  #number of database shards
 provdb_engine="ofi+tcp;ofi_rxm"  #the OFI libfabric provider used for the Mochi stack
@@ -36,6 +37,7 @@ provdb_domain=mlx5_0 #only needed for verbs provider   <------------ ***SET ME (
 ####################################
 #Options for the parameter server
 ####################################
+use_pserver=1 #enable or disable the pserver
 pserver_extra_args="" #any extra command line arguments to pass
 pserver_port=5559  #port for parameter server
 pserver_nt=2 #number of worker threads

scripts/launch/run_services.sh

@@ -90,7 +90,7 @@ echo "Chimbuko Services: Launching Chimbuko services on interface ${service_node
 #Provenance database
 extra_args=${ad_extra_args}
 ps_extra_args=${pserver_extra_args}
-if (( 1 )); then
+if (( ${use_provdb} == 1 )); then
     echo "==========================================="
     echo "Instantiating provenance database"
     echo "==========================================="
@@ -126,10 +126,19 @@ if (( 1 )); then
     ps_extra_args+=" -provdb_addr \"${prov_add}\""
     echo "Chimbuko Services: Enabling provenance database with arg: ${extra_args}"
     cd -
+else
+    echo "Chimbuko Services: Provenance database is not in use, provenance data will be stored in ASCII format at ${provdb_writedir}"
+    extra_args+=" -prov_outputpath ${provdb_writedir}"
+    ps_extra_args+=" -prov_outputpath ${provdb_writedir}"
 fi
 
 #Visualization
-if (( ${use_viz} )); then
+if (( ${use_viz} == 1 )); then
+    if (( ${use_pserver} != 1 )); then
+	echo "Chimbuko Services: Error - cannot use viz without the pserver"
+	exit 1
+    fi
+
     cd ${viz_dir}
     HOST=`hostname`
     export SHARDED_NUM=${provdb_nshards}
@@ -186,7 +195,7 @@ if (( ${use_viz} )); then
     echo "Chimbuko Services: Webserver is running on ${HOST}:${viz_port} and is ready for the user to connect"
 fi
 
-if (( 1 )); then
+if (( ${use_pserver} == 1 )); then
     pserver_addr="tcp://${ip}:${pserver_port}"  #address for parameter server in format "tcp://IP:PORT"    
 
     echo "==========================================="
@@ -227,10 +236,11 @@ echo "Or equivalent"
 echo ${ad_opts} > ${var_dir}/chimbuko_ad_opts.var
 
 
-
-#Wait for pserver to exit, which means it's time to end the viz
-echo "Chimbuko Services: waiting for pserver (pid ${ps_pid}) to terminate"
-wait ${ps_pid}
+if (( ${use_pserver} == 1 )); then
+    #Wait for pserver to exit, which means it's time to end the viz
+    echo "Chimbuko Services: waiting for pserver (pid ${ps_pid}) to terminate"
+    wait ${ps_pid}
+fi
 
 if (( ${use_viz} == 1 )); then
     echo "Chimbuko Services: Terminating Chimbuko visualization"
@@ -259,4 +269,10 @@ if (( ${use_viz} == 1 )); then
     cd -
 fi
 
+if (( ${use_provdb} == 1 )); then
+    echo "Chimbuko Services: waiting for provdb (pid ${provdb_pid}) to terminate"
+    wait ${provdb_pid}
+fi
+
+
 echo "Chimbuko Services: Service script complete" $(date)