Services script now outputs full viz url to a file in the var dir

Pserver simulation can now optionally send data to a remote viz instance
Simulator example3 now optionally allows send to viz and also sleeps between io steps to simulate a real run

Author: giltirn

scripts/launch/run_services.sh

@@ -159,8 +159,8 @@ fi
 #Visualization
 if (( ${use_viz} == 1 )); then
     if (( ${use_pserver} != 1 )); then
-	echo "Chimbuko Services: Error - cannot use viz without the pserver"
-	exit 1
+    	echo "Chimbuko Services: Error - cannot use viz without the pserver"
+    	exit 1
     fi
     if (( ${use_provdb} == 0 )); then
 	echo "Chimbuko Services: Error - cannot use viz without the provDB"
@@ -230,6 +230,7 @@ if (( ${use_viz} == 1 )); then
 
     echo $HOST > ${var_dir}/chimbuko_webserver.host
     echo $viz_port > ${var_dir}/chimbuko_webserver.port
+    echo $ws_addr > ${var_dir}/chimbuko_webserver.url
     echo "Chimbuko Services: Webserver is running on ${HOST}:${viz_port} and is ready for the user to connect"
 fi
 

sim/include/sim/pserver.hpp

@@ -1,6 +1,7 @@
 #pragma once
 #include <chimbuko_config.h>
 #include<chimbuko/param.hpp>
+#include<chimbuko/util/curlJsonSender.hpp>
 #include<chimbuko/pserver/GlobalAnomalyStats.hpp>
 #include<chimbuko/pserver/GlobalCounterStats.hpp>
 #include<chimbuko/pserver/GlobalAnomalyMetrics.hpp>
@@ -20,6 +21,7 @@ namespace chimbuko_sim{
     PSstatSenderGlobalAnomalyMetricsPayload anomaly_metrics_payload;
     LocalNet m_net; /**< The net server */
     ParamInterface* m_ad_params; /**< If using an AD algorithm to analyze the trace, these are the global parameters*/
+    curlJsonSender* m_viz; /**< Curl connection to visualization (optional, default disabled)*/
   public:
 
     /**
@@ -50,6 +52,11 @@ namespace chimbuko_sim{
       global_anomaly_metrics.add(loc);
     }
 
+    /**
+     * @brief Enable streaming output to be written to the viz on the given url
+     */
+    void enableVizOutput(const std::string &url);
+
     /**
      * @brief Mirror write functionality of PSstatSender
      */

sim/main/example3.cpp

@@ -14,6 +14,7 @@ int main(int argc, char **argv){
   MPI_Init(&argc, &argv);
 #endif
 
+  int sim_step_freq_ms = 1000; //frequency with which we iterate over io step (independent of actual function timings)
   provDBsetup pdb_setup;
   int i=1;
   while(i<argc){
@@ -25,6 +26,14 @@ int main(int argc, char **argv){
       pdb_setup.remote_server_instances = std::stoi(argv[i+3]);
       pdb_setup.use_local = false;
       i+=4;
+    }else if(sarg == "-enable_viz"){
+      if(i+1 >= argc) fatal_error("Not enough arguments provided");
+      getPserver().enableVizOutput(argv[i+1]);
+      i+=2;
+    }else if(sarg == "-sim_step_freq"){
+      if(i+1 >= argc) fatal_error("Not enough arguments provided");
+      sim_step_freq_ms = std::stoi(argv[i+1]);
+      i+=2;
     }else{
       fatal_error(stringize("Unknown argument: %s",argv[i]));
     }
@@ -101,6 +110,7 @@ int main(int argc, char **argv){
     }
     //PServer dump its output
     getPserver().writeStreamingOutput();
+    std::this_thread::sleep_for(std::chrono::milliseconds(sim_step_freq_ms));
   }
 
 #ifdef USE_MPI

sim/main/run_example3_pdb_ext/chimbuko_config.sh

@@ -8,7 +8,7 @@ service_node_iface=eth0 #network interface upon which communication to the servi
 ####################################
 #Options for visualization module
 ####################################
-use_viz=0 #enable or disable the visualization
+use_viz=1 #enable or disable the visualization
 viz_root=/opt/chimbuko/viz    #the root directory of the visualization module  <------------ ***SET ME (if using viz)***
 viz_worker_port=6379  #the port on which to run the redis server for the visualization backend
 viz_port=5002 #the port on which to run the webserver
@@ -42,7 +42,7 @@ export FI_OFI_RXM_USE_SRX=1 # use shared recv context in RXM; should improve sca
 ####################################
 #Options for the parameter server
 ####################################
-use_pserver=0 #enable or disable the pserver
+use_pserver=1 #enable or disable the pserver
 pserver_extra_args="" #any extra command line arguments to pass
 pserver_port=5559  #port for parameter server
 pserver_nt=2 #number of worker threads

sim/main/run_example3_pdb_ext/run.sh

@@ -13,10 +13,12 @@ if (( 1 )); then
     ad_cmd=$(cat chimbuko/vars/chimbuko_ad_cmdline.var)
 fi
 
+ws_addr=$(cat chimbuko/vars/chimbuko_webserver.url)
+
 #Run the simulation 
 if (( 1 )); then
     echo "Running sim"
-    ./example3 -remote_provdb chimbuko/provdb ${provdb_nshards} ${provdb_ninstances} 2>&1 | tee chimbuko/logs/sim.log
+    ./example3 -remote_provdb chimbuko/provdb ${provdb_nshards} ${provdb_ninstances} -enable_viz ${ws_addr} 2>&1 | tee chimbuko/logs/sim.log
 fi
 
 wait

sim/src/pserver.cpp

@@ -17,11 +17,12 @@ void pserverSim::writeStreamingOutput() const{
 
   if(json_packet.size() != 0){ //only write if there is anything to write! (it writes "null" to the file otherwise)
     std::ofstream out(fname.str()); out << json_packet.dump(4);
+    if(m_viz) m_viz->send(json_packet);
   }
 }
 
 pserverSim::pserverSim(): anomaly_stats_payload(&global_func_stats), counter_stats_payload(&global_counter_stats), 
-			  anomaly_metrics_payload(&global_anomaly_metrics), m_ad_params(nullptr){
+			  anomaly_metrics_payload(&global_anomaly_metrics), m_ad_params(nullptr), m_viz(nullptr){
   if(adAlgorithmParams().algorithm != "none"){
     m_ad_params = ParamInterface::set_AdParam(adAlgorithmParams().algorithm);
     m_net.add_payload(new NetPayloadUpdateParams(m_ad_params, false));
@@ -30,4 +31,11 @@ pserverSim::pserverSim(): anomaly_stats_payload(&global_func_stats), counter_sta
 
 pserverSim::~pserverSim(){
   if(m_ad_params) delete m_ad_params;
+  if(m_viz) delete m_viz;
+}
+
+void pserverSim::enableVizOutput(const std::string &url){
+  std::cout << "pserverSim enabling viz output on url " << url << std::endl;
+  if(m_viz) delete m_viz;
+  m_viz = new curlJsonSender(url);
 }