provDB commit freq is now an argument in chimbuko_config.sh

If commit freq is 0 the committer is now disabled
sonata_profile_parse.pl can now parse profile data from margo state dumps
provDB benchmark changes:
	Updated schema
	Can now request server does a margo state dump periodically
	Now outputs periodic perf data alongside full run perf stats
	Can now change perf write freq and directory from cmdline
	Perf info is now written by all ranks

Author: giltirn

app/provdb_commit.cpp

@@ -45,6 +45,11 @@ int main(int argc, char** argv){
   int nshards = std::stoi(argv[2]);
   int freq_ms = std::stoi(argv[3]);
 
+  if(freq_ms == 0){
+    progressStream << "ProvDB Commit freq==0, I am not needed. Done" << std::endl;
+    return 0;
+  } 
+
   std::string protocol = ADProvenanceDBengine::getProtocolFromAddress(addr);
   if(ADProvenanceDBengine::getProtocol().first != protocol){
     int mode = ADProvenanceDBengine::getProtocol().second;

app/sonata_profile_parse.pl

@@ -1,12 +1,22 @@
 #!/usr/bin/perl
 
 $ARGC=scalar @ARGV;
-if($ARGC != 1){
-    print "Usage: <script.pl> <filename.csv>\n";
+if($ARGC < 1){
+    print "Usage: <script.pl> <filename.csv> <options>\n";
+    print "Options:\n";
+    print "         -is_state_dump   The file is a margo state dump\n";
     exit -1;
 }
 
 $file = $ARGV[0];
+
+$is_state_dump=0;
+for($i=1;$i<$ARGC;$i++){
+    if($ARGV[$i] == "-is_state_dump"){ #file is generated by a margo state dump
+	$is_state_dump = 1;
+    }
+}
+
 open(IN, $file);
 
 #cf https://xgitlab.cels.anl.gov/sds/margo/blob/master/doc/instrumentation.md#generating-a-profile-and-topology-graph
@@ -15,54 +25,71 @@
 $i=0;
 $nrpc;
 %rpcnames = ();
+$active = 1;
+if($is_state_dump == 1){ #not active from the start
+    $active = 0;
+}
 
 while(<IN>){
     $line=$_;
-    if($i==0){
-	if(!($line=~m/(\d+)/)){
-	    print "Could not parse number of RPC names from line $line\n";
-	    exit -1;
-	}else{
-	    $nrpc = $1;
-	    print "$nrpc RPCs\n";
-	}
-    }elsif($i==1){
-	if(!($line=~m/\d+\,(.*)/)){
-	    print "Could not parse server IP from line $line\n";
-	    exit -1;
-	}else{
-	    $ip = $1;
-	    print "IP $ip\n";
-	}
-    }elsif($i>=2 && $i<2+$nrpc){
-	if(!($line=~m/(\w+)\,(.*)/)){
-	    print "Could not parse RPC hash/name from line $line\n";
-	    exit -1;
-	}else{
-	    $hash = $1;
-	    $name = $2;
-	    print "$name -> $hash\n";
-	    $rpcnames{"$hash"} = $name;
-	}
-    }elsif( ($i -2 - $nrpc) % 2 == 0){
-	#Even lines format is hash, avg, rpc_breadcrumb, addr_hash, origin_or_target, cumulative, _min, _max, count, abt_pool_size_hwm, abt_pool_size_lwm, abt_pool_size_cumulative, abt_pool_total_size_hwm, abt_pool_total_size_lwm, abt_pool_total
 
-	@vals = split('\s*,\s*', $line);
-	if(scalar @vals != 15){
-	    print "Could not parse data line $line\n";
-	    exit -1;
-	}
-	$hash = $vals[0];
-	if(!(exists $rpcnames{$hash})){
-	    print "Could not find hash '$hash' in map\n";
-	    exit -1;
-	}
-	$name = $rpcnames{$hash};
-	$avg = $vals[1];
-	$count = $vals[8];
-	$total = $vals[5];
-	$loc = $vals[4];
-	print "Func $name Origin/Target $loc Avg $avg Count $count Total $total\n";
+    if($active == 1){
+    	if($is_state_dump == 1 && $line=~m/^$/){
+    	    $active = 0;
+    	}elsif($i==0){
+    	    if(!($line=~m/(\d+)/)){
+    		print "Could not parse number of RPC names from line $line\n";
+    		exit -1;
+    	    }else{
+    		$nrpc = $1;
+    		print "$nrpc RPCs\n";
+    	    }
+    	}elsif($i==1){
+    	    if(!($line=~m/\d+\,(.*)/)){
+    		print "Could not parse server IP from line $line\n";
+    		exit -1;
+    	    }else{
+    		$ip = $1;
+    		print "IP $ip\n";
+    	    }
+    	}elsif($i>=2 && $i<2+$nrpc){
+    	    if(!($line=~m/(\w+)\,(.*)/)){
+    		print "Could not parse RPC hash/name from line $line\n";
+    		exit -1;
+    	    }else{
+    		$hash = $1;
+    		$name = $2;
+    		print "$name -> $hash\n";
+    		$rpcnames{"$hash"} = $name;
+    	    }
+    	}elsif( ($i -2 - $nrpc) % 2 == 0){
+    	    #Even lines format is hash, avg, rpc_breadcrumb, addr_hash, origin_or_target, cumulative, _min, _max, count, abt_pool_size_hwm, abt_pool_size_lwm, abt_pool_size_cumulative, abt_pool_total_size_hwm, abt_pool_total_size_lwm, abt_pool_total
+
+    	    @vals = split('\s*,\s*', $line);
+    	    if(scalar @vals != 15){
+    		print "Could not parse data line $line\n";
+    		exit -1;
+    	    }
+    	    $hash = $vals[0];
+    	    if(!(exists $rpcnames{$hash})){
+    		print "Could not find hash '$hash' in map\n";
+    		exit -1;
+    	    }
+    	    $name = $rpcnames{$hash};
+    	    $avg = $vals[1];
+    	    $count = $vals[8];
+    	    $total = $vals[5];
+    	    $loc = $vals[4];
+    	    print "Func $name Origin/Target $loc Avg $avg Count $count Total $total\n";
+    	}
+    	$i++;
+    }elsif($line=~m/Margo RPC profiling/){
+    	#read 6 lines
+    	for($ii=0;$ii<6;$ii++){
+    	    <IN>;
+    	}
+    	$active = 1;
     }
-    $i++;
 }
+
+close(IN);