Provdb now writes a map of shard index to instance and provider indices for use by the viz

Updated config scripts for benchmark_suite func_multimodal and c_from_python
run_services changes:
	Added viz initialization parameters for multi-instance provDB
	Committer log output now piped to file rather than through tee to reduce spam
	Viz initialization now reports error if provDB is disabled

Author: giltirn

app/provdb_admin.cpp

@@ -402,6 +402,9 @@ int main(int argc, char** argv) {
 	    PSprogressStream << "initialized global DB collections" << std::endl;
 	  }
 
+	  //Instance 0 writes a map of shard index to server instance and provider index for use by the viz
+	  if(instance == 0) setup.writeShardInstanceMap();
+
 	  //Write address to file; do this after initializing collections so that the existence of the file can be used to signal readiness
 	  {	    
 	    std::ofstream f(setup.getInstanceAddressFilename(instance));

benchmark_suite/c_from_python/chimbuko_config.sh

@@ -26,15 +26,18 @@ chimbuko_services="infer" #The location of the Chimbuko service script. If set t
 use_provdb=1 #enable or disable the provDB. If disabled the provenance data will be written as JSON ASCII into the ${provdb_writedir} set below
 provdb_extra_args="" #any extra command line arguments to pass
 provdb_nshards=4  #number of database shards
+provdb_ninstances=1 #number of database server instances. Shards are distributed over instances
 provdb_engine="ofi+tcp;ofi_rxm"  #the OFI libfabric provider used for the Mochi stack
 provdb_port=5000 #the port of the provenance database
-provdb_nthreads=4 #number of worker threads; should be >= the number of shards
 provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default
 provdb_commit_freq=10000   #frequency ms at which the provenance database is committed to disk. If set to 0 it will commit only at the end
 
 #With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
 provdb_domain=mlx5_0 #only needed for verbs provider   <------------ ***SET ME (if using verbs)***
 
+export FI_UNIVERSE_SIZE=1600  # Defines the expected number of provenance DB clients per instance <------------- *** SET ME (should be larger than the number of clients/instance)
+export FI_MR_CACHE_MAX_COUNT=0   # disable MR cache in libfabric; still problematic as of libfabric 1.10.1
+export FI_OFI_RXM_USE_SRX=1 # use shared recv context in RXM; should improve scalability
 
 ####################################
 #Options for the parameter server

benchmark_suite/func_multimodal/chimbuko_config.sh

@@ -26,15 +26,18 @@ chimbuko_services="infer" #The location of the Chimbuko service script. If set t
 use_provdb=1 #enable or disable the provDB. If disabled the provenance data will be written as JSON ASCII into the ${provdb_writedir} set below
 provdb_extra_args="" #any extra command line arguments to pass
 provdb_nshards=4  #number of database shards
+provdb_ninstances=1 #number of database server instances. Shards are distributed over instances
 provdb_engine="ofi+tcp;ofi_rxm"  #the OFI libfabric provider used for the Mochi stack
 provdb_port=5000 #the port of the provenance database
-provdb_nthreads=4 #number of worker threads; should be >= the number of shards
 provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default
 provdb_commit_freq=10000   #frequency ms at which the provenance database is committed to disk. If set to 0 it will commit only at the end
 
 #With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
 provdb_domain=mlx5_0 #only needed for verbs provider   <------------ ***SET ME (if using verbs)***
 
+export FI_UNIVERSE_SIZE=1600  # Defines the expected number of provenance DB clients per instance <------------- *** SET ME (should be larger than the number of clients/instance)
+export FI_MR_CACHE_MAX_COUNT=0   # disable MR cache in libfabric; still problematic as of libfabric 1.10.1
+export FI_OFI_RXM_USE_SRX=1 # use shared recv context in RXM; should improve scalability
 
 ####################################
 #Options for the parameter server
@@ -98,6 +101,11 @@ TAU_ADIOS2_FILE_PREFIX=tau-metrics  #the prefix of tau adios2 files; full filena
 export TAU_ADIOS2_FILENAME="${TAU_ADIOS2_PATH}/${TAU_ADIOS2_FILE_PREFIX}"
 
 if [[ ${backend_root} == "infer" ]]; then
+    if [[ $(which provdb_admin) == "" ]]; then
+	echo "When inferring the backend root directory, could not find provdb_admin in PATH. Please add your Chimbuko bin directory to PATH"
+	exit 1
+    fi
+    
     backend_root=$( readlink -f $(which provdb_admin | sed 's/provdb_admin//')/../ )
 fi
 

include/chimbuko/provdb/setup.hpp

@@ -124,7 +124,24 @@ namespace chimbuko{
     static inline std::string getGlobalDBname(){
       return "provdb.global";
     }
-	
+
+
+    /**
+     * @brief Write a file containing a map of shard index to server instance and provider index
+     * @param dir The directory to write the file to
+     *
+     * Line format is: <shard index> <server instance> <provider index>
+     */
+    inline void writeShardInstanceMap(const std::string &dir = "."){
+      std::string filename = dir + "/provider.map";
+      std::ofstream out(filename);
+      for(int shard=0;shard<m_nshards;shard++){
+	out << shard << " " << getShardInstance(shard) << " " << getShardProviderIndex(shard) << std::endl;
+      }
+      out << "global " << getGlobalDBinstance() << " " << getGlobalDBproviderIndex() << std::endl;
+      if(!out.good()) fatal_error("Could not write shard instance map");
+    }
+
 
   };
 

scripts/launch/run_services.sh

@@ -147,7 +147,7 @@ if (( ${use_provdb} == 1 )); then
     echo "==========================================="
     for((i=0;i<provdb_ninstances;i++)); do
 	echo "Chimbuko services launching provDB committer ${i} of ${provdb_ninstances}"
-	provdb_commit "${provdb_dir}" -instance ${i} -ninstances ${provdb_ninstances} -nshards ${provdb_nshards} -freq_ms ${provdb_commit_freq} 2>&1 | tee ${log_dir}/committer_${i}.log &
+	provdb_commit "${provdb_dir}" -instance ${i} -ninstances ${provdb_ninstances} -nshards ${provdb_nshards} -freq_ms ${provdb_commit_freq} > ${log_dir}/committer_${i}.log 2>&1 &
 	sleep 1
     done
     sleep 3
@@ -159,22 +159,32 @@ if (( ${use_viz} == 1 )); then
 	echo "Chimbuko Services: Error - cannot use viz without the pserver"
 	exit 1
     fi
-    if ((provdb_ninstances > 1)); then
-	echo "Chimbuko Services: Error - viz currently does not support multiple provDB instances"
+    if (( ${use_provdb} == 0 )); then
+	echo "Chimbuko Services: Error - cannot use viz without the provDB"
 	exit 1
     fi
 
-    cd ${viz_dir}
-    HOST=`hostname`
+    #Provide parameters for provenance database
+    export PROVDB_NINSTANCE=${provdb_ninstances}
     export SHARDED_NUM=${provdb_nshards}
-    export PROVDB_ADDR=${prov_add}
+    export PROVENANCE_DB=${provdb_writedir} #already an absolute path
+
+    if (( ${provdb_ninstances} == 1 )); then
+	#Simpler instantiation if a single server
+	export PROVDB_ADDR=$(cat ${provdb_dir}/provider.address.0)
+	echo "Chimbuko Services: viz is connecting to provDB provider 0 on address" $PROVDB_ADDR
+    else
+	export PROVDB_ADDR_PATH=$(readlink -f ${provdb_dir})
+	echo "Chimbuko Services: viz is obtaining provDB addresses from path" $PROVDB_ADDR_PATH
+    fi
 
+    cd ${viz_dir}
+    HOST=`hostname`
     export SERVER_CONFIG="production"
     export DATABASE_URL="sqlite:///${viz_dir}/main.sqlite"
     export ANOMALY_STATS_URL="sqlite:///${viz_dir}/anomaly_stats.sqlite"
     export ANOMALY_DATA_URL="sqlite:///${viz_dir}/anomaly_data.sqlite"
     export FUNC_STATS_URL="sqlite:///${viz_dir}/func_stats.sqlite"
-    export PROVENANCE_DB=${provdb_writedir}
     export CELERY_BROKER_URL="redis://${HOST}:${viz_worker_port}"
 
     #Setup redis
@@ -192,7 +202,7 @@ if (( ${use_viz} == 1 )); then
     cd ${viz_root}
 
     echo "Chimbuko Services: create db ..."
-    python3 manager.py createdb
+    python3 -u manager.py createdb
 
     echo "Chimbuko Services: run redis ..."
     redis-server ${viz_dir}/redis.conf