Added a command line option -db_write_dir to the provDB to set the write directory for the database shards

The write directory of the provDB shards can now be specified by changing provdb_writedir in chimbuko_config script
run_services script now fails if it tries to use a variable that has not been set

Author: giltirn

app/provdb_admin.cpp

@@ -86,8 +86,9 @@ struct ProvdbArgs{
   int nthreads;
   std::string db_type;
   unsigned long db_commit_freq;
+  std::string db_write_dir;
 
-  ProvdbArgs(): engine("ofi+tcp"), autoshutdown(true), nshards(1), db_type("unqlite"), nthreads(1), db_commit_freq(10000){}
+  ProvdbArgs(): engine("ofi+tcp"), autoshutdown(true), nshards(1), db_type("unqlite"), nthreads(1), db_commit_freq(10000), db_write_dir("."){}
 };
 
 
@@ -112,6 +113,7 @@ int main(int argc, char** argv) {
     addOptionalCommandLineArg(parser, nthreads, "Specify the number of RPC handler threads (default 1)");
     addOptionalCommandLineArg(parser, db_type, "Specify the Sonata database type (default \"unqlite\")");
     addOptionalCommandLineArg(parser, db_commit_freq, "Specify the frequency at which the database flushes to disk in ms (default 10000)");
+    addOptionalCommandLineArg(parser, db_write_dir, "Specify the directory in which the database shards will be written (default \".\")");
 
     if(argc-1 < parser.nMandatoryArgs() || (argc == 2 && std::string(argv[1]) == "-help")){
       parser.help(std::cout);
@@ -167,15 +169,15 @@ int main(int argc, char** argv) {
 	sonata::Admin admin(engine);
 	progressStream << "ProvDB Admin: creating global data database" << std::endl;
 	std::string glob_db_name = "provdb.global";
-	std::string glob_db_config = stringize("{ \"path\" : \"./%s.unqlite\" }", glob_db_name.c_str());
+	std::string glob_db_config = stringize("{ \"path\" : \"%s/%s.unqlite\" }", args.db_write_dir.c_str(), glob_db_name.c_str());
 	admin.createDatabase(addr, 0, glob_db_name, args.db_type, glob_db_config);
 
 	progressStream << "ProvDB Admin: creating " << args.nshards << " database shards" << std::endl;
 
 	std::vector<std::string> db_shard_names(args.nshards);
 	for(int s=0;s<args.nshards;s++){
 	  std::string db_name = stringize("provdb.%d",s);
-	  std::string config = stringize("{ \"path\" : \"./%s.unqlite\" }", db_name.c_str());
+	  std::string config = stringize("{ \"path\" : \"%s/%s.unqlite\" }", args.db_write_dir.c_str(), db_name.c_str());
 	  progressStream << "ProvDB Admin: Shard " << s << ": " << db_name << " " << config << " " << args.db_type << std::endl;
 	  admin.createDatabase(addr, 0, db_name, args.db_type, config);
 	  db_shard_names[s] = db_name;

scripts/launch/chimbuko_config.sh

@@ -27,6 +27,7 @@ provdb_nshards=4  #number of database shards
 provdb_engine="ofi+tcp;ofi_rxm"  #the OFI libfabric provider used for the Mochi stack
 provdb_port=5000 #the port of the provenance database
 provdb_nthreads=4 #number of worker threads; should be >= the number of shards
+provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default
 
 #With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
 provdb_domain=mlx5_0 #only needed for verbs provider   <------------ ***SET ME (if using verbs)***

scripts/launch/run_services.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 set -e
+set -u
 set -o pipefail
 
 #---------------------------------------------------------------------------
@@ -46,7 +47,7 @@ fi
 
 mkdir chimbuko
 mkdir chimbuko/logs
-mkdir chimbuko/provdb
+mkdir chimbuko/provdb    #directory in which provDB is run. Also the default directory for the database shards
 mkdir chimbuko/viz
 mkdir chimbuko/vars      #directory for files containing variables for inter-script communication
 mkdir chimbuko/adios2
@@ -93,15 +94,21 @@ if (( 1 )); then
     echo "==========================================="
     echo "Instantiating provenance database"
     echo "==========================================="
+
+    #If provdb_writedir is a relative path it should be relative to the run directory
+    #Turn it into an absolute path before entering the subdirectory
+    provdb_writedir=$(readlink -f ${provdb_writedir})
+
     cd ${provdb_dir}
-    rm -f provdb.*.unqlite*  provider.address
+
+    rm -f ${provdb_writedir}/provdb.*.unqlite*  provider.address
 
     provdb_addr="${service_node_iface}:${provdb_port}"    #can be IP:PORT or ADAPTOR:PORT per libfabric conventions
     if [[ ${provdb_engine} == "verbs" ]]; then
 	provdb_addr="${provdb_domain}/${provdb_addr}"
     fi
 
-    provdb_admin "${provdb_addr}" ${provdb_extra_args} -engine ${provdb_engine} -nshards ${provdb_nshards} -nthreads ${provdb_nthreads} 2>&1 | tee ${log_dir}/provdb.log &
+    provdb_admin "${provdb_addr}" ${provdb_extra_args} -engine ${provdb_engine} -nshards ${provdb_nshards} -nthreads ${provdb_nthreads} -db_write_dir ${provdb_writedir} 2>&1 | tee ${log_dir}/provdb.log &
     provdb_pid=$!
 
     start_time=$SECONDS
@@ -134,7 +141,7 @@ if (( ${use_viz} )); then
     export ANOMALY_STATS_URL="sqlite:///${viz_dir}/anomaly_stats.sqlite"
     export ANOMALY_DATA_URL="sqlite:///${viz_dir}/anomaly_data.sqlite"
     export FUNC_STATS_URL="sqlite:///${viz_dir}/func_stats.sqlite"
-    export PROVENANCE_DB=${provdb_dir}
+    export PROVENANCE_DB=${provdb_writedir}
     export CELERY_BROKER_URL="redis://${HOST}:${viz_worker_port}"
 
     #Setup redis
@@ -181,12 +188,13 @@ if (( ${use_viz} )); then
 fi
 
 if (( 1 )); then
+    pserver_addr="tcp://${ip}:${pserver_port}"  #address for parameter server in format "tcp://IP:PORT"    
+
     echo "==========================================="
     echo "Chimbuko Services: Instantiating pserver"
     echo "==========================================="
     echo "Chimbuko Services: Pserver $pserver_addr"
 
-    pserver_addr="tcp://${ip}:${pserver_port}"  #address for parameter server in format "tcp://IP:PORT"    
     pserver -nt ${pserver_nt} -logdir ${log_dir} -port ${pserver_port} ${ps_extra_args} 2>&1 | tee ${log_dir}/pserver.log  &
 
     ps_pid=$!