Skip to content

Commit c1089ee

Browse files
giltirngiltirn
committed
Updated run script and config for benchmark_provdb to use service launcher
1 parent b4d20de commit c1089ee

2 files changed

Lines changed: 138 additions & 30 deletions

File tree

‎benchmark_suite/benchmark_provdb/chimbuko_config.sh‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
#Note: This configuration file is sourced into the bash environment for Chimbuko startup scripts, thus the user must follow correct shell conventions
2+
#Please do not remove any of the variables!
3+
4+
#IMPORTANT NOTE: Variables that cannot be left as default are marked as <------------ ***SET ME***
5+
export MARGO_ENABLE_PROFILING=1
6+
export HG_NA_LOG_LEVEL=error
7+
export HG_LOG_LEVEL=debug
8+
9+
service_node_iface=eth0 #network interface upon which communication to the service node is performed <------------ ***SET ME***
10+
11+
####################################
12+
#Options for visualization module
13+
####################################
14+
use_viz=0 #enable or disable the visualization
15+
viz_root=/opt/chimbuko/viz #the root directory of the visualization module <------------ ***SET ME (if using viz)***
16+
viz_worker_port=6379 #the port on which to run the redis server for the visualization backend
17+
viz_port=5002 #the port on which to run the webserver
18+
export C_FORCE_ROOT=1 #required only for docker runs, allows celery to execute properly as root user <----------------- *** SET ME (if using Docker)
19+
20+
############################################################
21+
#General options for Chimbuko backend (pserver, ad, provdb)
22+
############################################################
23+
backend_root="infer" #The root install directory of the PerformanceAnalysis libraries. If set to "infer" it will be inferred from the path of the executables
24+
25+
####################################
26+
#Options for the provenance database
27+
####################################
28+
use_provdb=1 #enable or disable the provDB. If disabled the provenance data will be written as JSON ASCII into the ${provdb_writedir} set below
29+
provdb_extra_args=""
30+
provdb_nshards=4 #number of database shards
31+
provdb_engine="ofi+tcp;ofi_rxm" #the OFI libfabric provider used for the Mochi stack
32+
provdb_port=5000 #the port of the provenance database
33+
provdb_nthreads=4 #number of worker threads; should be >= the number of shards
34+
provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default
35+
provdb_commit_freq=10000
36+
37+
#With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node)
38+
provdb_domain=mlx5_0 #only needed for verbs provider <------------ ***SET ME (if using verbs)***
39+
40+
41+
####################################
42+
#Options for the parameter server
43+
####################################
44+
use_pserver=0 #enable or disable the pserver
45+
pserver_extra_args="-stat_outputdir chimbuko/viz" #any extra command line arguments to pass
46+
pserver_port=5559 #port for parameter server
47+
pserver_nt=2 #number of worker threads
48+
49+
####################################
50+
#Options for the AD module
51+
####################################
52+
ad_extra_args="-perf_outputpath chimbuko/logs -perf_step 1" #any extra command line arguments to pass. chimbuko/logs is automatically created by services script
53+
ad_win_size=5 #number of events around an anomaly to store; provDB entry size is proportional to this
54+
ad_alg="hbos" #the anomaly detection algorithm. Valid values are "hbos" and "sstd"
55+
ad_outlier_hbos_threshold=0.99 #the percentile of events outside of which are considered anomalies by the HBOS algorithm
56+
ad_outlier_sstd_sigma=12 #number of standard deviations that defines an outlier in the SSTD algorithm
57+
58+
####################################
59+
#Options for TAU
60+
####################################
61+
export TAU_ADIOS2_ENGINE=SST #online communication engine (alternative BP4 although this goes through the disk system and may be slower unless the BPfiles are stored on a burst disk)
62+
export TAU_ADIOS2_ONE_FILE=FALSE #a different connetion file for each rank
63+
export TAU_ADIOS2_PERIODIC=1 #enable/disable ADIOS2 periodic output
64+
export TAU_ADIOS2_PERIOD=1000000 #period in us between ADIOS2 io steps
65+
export TAU_THREAD_PER_GPU_STREAM=1 #force GPU streams to appear as different TAU virtual threads
66+
export TAU_THROTTLE=0 #enable/disable throttling of short-running functions
67+
68+
export TAU_MAKEFILE=/opt/tau2/x86_64/lib/Makefile.tau-papi-mpi-pthread-pdt-adios2 #The TAU makefile to use <------------ ***SET ME***
69+
TAU_EXEC="tau_exec -T papi,mpi,pthread,pdt,adios2 -adios2_trace" #how to execute tau_exec; the -T arguments should mirror the makefile name <------------ ***SET ME***
70+
71+
export EXE_NAME=main #the name of the executable (without path) <------------ ***SET ME***
72+
73+
TAU_ADIOS2_PATH=chimbuko/adios2 #path where the adios2 files are to be stored. Chimbuko services creates the directory chimbuko/adios2 in the working directory and this should be used by default
74+
TAU_ADIOS2_FILE_PREFIX=tau-metrics #the prefix of tau adios2 files; full filename is ${TAU_ADIOS2_PREFIX}-${EXE_NAME}-${RANK}.bp
75+
76+
77+
78+
79+
80+
81+
82+
83+
84+
85+
86+
87+
88+
###########################################################################
89+
# NON-USER VARIABLES BELOW = DON'T MODIFY THESE!!
90+
###########################################################################
91+
#Extra processing
92+
export TAU_ADIOS2_FILENAME="${TAU_ADIOS2_PATH}/${TAU_ADIOS2_FILE_PREFIX}"
93+
94+
if [[ ${backend_root} == "infer" ]]; then
95+
backend_root=$( readlink -f $(which provdb_admin | sed 's/provdb_admin//')/../ )
96+
97+
#Check run script exists
98+
if [ ! -f "${backend_root}/scripts/launch/run_services.sh" ]; then
99+
echo "Could not infer backend root directory: service script does not exist!"
100+
exit 1
101+
fi
102+
fi
103+

‎benchmark_suite/benchmark_provdb/run.sh‎

Lines changed: 35 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,40 @@
11
#!/bin/bash
2-
ranks=64
3-
cycles=60
2+
set -e
3+
4+
rm -rf chimbuko
5+
export CHIMBUKO_CONFIG=chimbuko_config.sh
6+
source ${CHIMBUKO_CONFIG}
7+
8+
if (( 1 )); then
9+
echo "Running services"
10+
${backend_root}/scripts/launch/run_services.sh 2>&1 | tee services.log &
11+
echo "Waiting"
12+
while [ ! -f chimbuko/vars/chimbuko_ad_cmdline.var ]; do sleep 1; done
13+
ad_cmd=$(cat chimbuko/vars/chimbuko_ad_cmdline.var)
14+
fi
15+
16+
provdb_addr=$(cat chimbuko/provdb/provider.address)
17+
exe=./benchmark_client
18+
cycles=100
19+
callstack_size=2
20+
ncounters=5
21+
winsize=5
22+
comm_messages_per_winevent=1
23+
anomalies_per_cycle=1
24+
normal_events_per_cycle=1
425
cycle_time_ms=1000
5-
anom_per_cyc=10
6-
norm_per_cyc=10
7-
shards=2
8-
threads=2
9-
#rm -f ps_perf.txt ps_perf_stats.txt
26+
nshards=${provdb_nshards}
27+
perf_write_freq=5 #cycles
28+
perf_dir=chimbuko/logs
29+
do_state_dump=0
30+
ranks=4
1031

11-
export CHIMBUKO_VERBOSE=1
12-
export MARGO_ENABLE_PROFILING=1
32+
client_cmd="$exe \"${provdb_addr}\" -cycles ${cycles} -callstack_size ${callstack_size} -ncounters ${ncounters} -winsize ${winsize} -comm_messages_per_winevent ${comm_messages_per_winevent} -anomalies_per_cycle ${anomalies_per_cycle} -normal_events_per_cycle ${normal_events_per_cycle} -cycle_time_ms ${cycle_time_ms} -nshards ${nshards} -perf_write_freq ${perf_write_freq} -perf_dir ${perf_dir} -do_state_dump ${do_state_dump} 2>&1 | tee chimbuko/logs/client.log"
1333

14-
rm -f provdb.*.unqlite* provider.address
15-
ip=$(hostname -i)
16-
provdb_addr=${ip}:5000
34+
if (( 1 )); then
35+
echo "Instantiating client"
36+
echo "Command is " $client_cmd
37+
eval "mpirun --allow-run-as-root -n ${ranks} ${client_cmd} &"
38+
fi
1739

18-
# -engine "na+sm"
19-
#-db_type "null"
20-
provdb_admin ${provdb_addr} -autoshutdown true -nshards ${shards} -nthreads ${threads} -db_commit_freq 10000 2>&1 | tee provdb.log &
21-
admin=$!
22-
23-
sleep 5
24-
provdb_inaddr=`cat provider.address`
25-
echo "Provdb addr ${provdb_inaddr}"
26-
27-
28-
mpirun --oversubscribe --allow-run-as-root -n ${ranks} ./benchmark_client ${provdb_inaddr} -cycles ${cycles} -cycle_time_ms ${cycle_time_ms} -anomalies_per_cycle ${anom_per_cyc} -normal_events_per_cycle ${norm_per_cyc} -nshards ${shards} 2>&1 | tee client.log
29-
30-
wait $admin
31-
32-
dir=${ranks}rank_cyc${cycle_time_ms}ms_anom${anom_per_cyc}_norm${norm_per_cyc}_shards${shards}_threads${threads}_multithread_prdcommit
33-
mkdir ${dir}
34-
mv client_stats.json profile* ${dir}
35-
echo "ADMIN is $admin"
40+
wait

0 commit comments

Comments
 (0)