|
| 1 | +#Note: This configuration file is sourced into the bash environment for Chimbuko startup scripts, thus the user must follow correct shell conventions |
| 2 | +#Please do not remove any of the variables! |
| 3 | + |
| 4 | +#IMPORTANT NOTE: Variables that cannot be left as default are marked as <------------ ***SET ME*** |
| 5 | + |
| 6 | +service_node_iface=eth0 #network interface upon which communication to the service node is performed <------------ ***SET ME*** |
| 7 | + |
| 8 | +#################################### |
| 9 | +#Options for visualization module |
| 10 | +#################################### |
| 11 | +use_viz=0 #enable or disable the visualization |
| 12 | +viz_root=/opt/chimbuko/viz #the root directory of the visualization module <------------ ***SET ME (if using viz)*** |
| 13 | +viz_worker_port=6379 #the port on which to run the redis server for the visualization backend |
| 14 | +viz_port=5002 #the port on which to run the webserver |
| 15 | +export C_FORCE_ROOT=1 #required only for docker runs, allows celery to execute properly as root user <----------------- *** SET ME (if using Docker) |
| 16 | + |
| 17 | +############################################################ |
| 18 | +#General options for Chimbuko backend (pserver, ad, provdb) |
| 19 | +############################################################ |
| 20 | +backend_root="infer" #The root install directory of the PerformanceAnalysis libraries. If set to "infer" it will be inferred from the path of the executables |
| 21 | +chimbuko_services="infer" #The location of the Chimbuko service script. If set to "infer" it will be inferred from backend_root |
| 22 | + |
| 23 | +#################################### |
| 24 | +#Options for the provenance database |
| 25 | +#################################### |
| 26 | +use_provdb=1 #enable or disable the provDB. If disabled the provenance data will be written as JSON ASCII into the ${provdb_writedir} set below |
| 27 | +provdb_extra_args="" #any extra command line arguments to pass |
| 28 | +provdb_nshards=4 #number of database shards |
| 29 | +provdb_ninstances=1 #number of database server instances. Shards are distributed over instances |
| 30 | +provdb_engine="ofi+tcp;ofi_rxm" #the OFI libfabric provider used for the Mochi stack |
| 31 | +provdb_port=5000 #the port of the provenance database |
| 32 | +provdb_writedir=chimbuko/provdb #the directory in which the provenance database is written. Chimbuko creates chimbuko/provdb which can be used as a default |
| 33 | +provdb_commit_freq=10000 #frequency ms at which the provenance database is committed to disk. If set to 0 it will commit only at the end |
| 34 | + |
| 35 | +#With "verbs" provider (used for infiniband, iWarp, etc) we need to also specify the domain, which can be found by running fi_info (on a compute node) |
| 36 | +provdb_domain=mlx5_0 #only needed for verbs provider <------------ ***SET ME (if using verbs)*** |
| 37 | + |
| 38 | +export FI_UNIVERSE_SIZE=1600 # Defines the expected number of provenance DB clients per instance <------------- *** SET ME (should be larger than the number of clients/instance) |
| 39 | +export FI_MR_CACHE_MAX_COUNT=0 # disable MR cache in libfabric; still problematic as of libfabric 1.10.1 |
| 40 | +export FI_OFI_RXM_USE_SRX=1 # use shared recv context in RXM; should improve scalability |
| 41 | + |
| 42 | +#################################### |
| 43 | +#Options for the parameter server |
| 44 | +#################################### |
| 45 | +use_pserver=0 #enable or disable the pserver |
| 46 | +pserver_extra_args="" #any extra command line arguments to pass |
| 47 | +pserver_port=5559 #port for parameter server |
| 48 | +pserver_nt=2 #number of worker threads |
| 49 | +#################################### |
| 50 | +#Options for the AD module |
| 51 | +#################################### |
| 52 | +ad_extra_args="-perf_outputpath chimbuko/logs -perf_step 1" #any extra command line arguments to pass. Note: chimbuko/logs is automatically created by services script |
| 53 | +ad_win_size=5 #number of events around an anomaly to store; provDB entry size is proportional to this so keep it small! |
| 54 | +ad_alg="hbos" #the anomaly detection algorithm. Valid values are "hbos" and "sstd" |
| 55 | +ad_outlier_hbos_threshold=0.99 #the percentile of events outside of which are considered anomalies by the HBOS algorithm |
| 56 | +ad_outlier_sstd_sigma=12 #number of standard deviations that defines an outlier in the SSTD algorithm |
| 57 | +#################################### |
| 58 | +#Options for TAU |
| 59 | +#Note: Only the TAU_ADIOS2_PATH, TAU_ADIOS2_FILE_PREFIX, EXE_NAME and TAU_ADIOS2_ENGINE variables are used by the Chimbuko services script and there only to generate the suggested |
| 60 | +# command to launch the AD (output to chimbuko/vars/chimbuko_ad_cmdline.var); they can be overridden by the run script if desired providing the appropriate modifications |
| 61 | +# are made to the AD launch command. The remainder of the variables are used only by TAU and can be freely overridden. |
| 62 | +#################################### |
| 63 | +export TAU_ADIOS2_ENGINE=SST #online communication engine (alternative BP4 although this goes through the disk system and may be slower unless the BPfiles are stored on a burst disk) |
| 64 | +export TAU_ADIOS2_ONE_FILE=FALSE #a different connection file for each rank |
| 65 | +export TAU_ADIOS2_PERIODIC=1 #enable/disable ADIOS2 periodic output |
| 66 | +export TAU_ADIOS2_PERIOD=1000000 #period in us between ADIOS2 io steps |
| 67 | +export TAU_THREAD_PER_GPU_STREAM=1 #force GPU streams to appear as different TAU virtual threads |
| 68 | +export TAU_THROTTLE=0 #enable/disable throttling of short-running functions |
| 69 | + |
| 70 | +export TAU_MAKEFILE=/opt/tau2/x86_64/lib/Makefile.tau-papi-mpi-pthread-pdt-adios2 #The TAU makefile to use <------------ ***SET ME*** |
| 71 | + |
| 72 | +#Note: the following 2 variables are not used by the service script but are included here for use from the user's run script allowing the application to be launched with either "${TAU_EXEC} <app>" or "${TAU_PYTHON} <app>" |
| 73 | +#Note: the "binding" -T ... is used by Tau to find the appropriate configuration. It can typically be inferred from the name of the Makefile. If using a non-MPI job the 'mpi' should be changed to 'serial' and a non-MPI build of |
| 74 | +# ADIOS2/TAU must exist |
| 75 | +#Suggestion: It is useful to test the command without Chimbuko first to ensure TAU picks up the correct binding; this can be done by 'export TAU_ADIOS2_ENGINE=BPFile' and then running the application with Tau but without Chimbuko. |
| 76 | +TAU_EXEC="tau_exec -T papi,mpi,pthread,pdt,adios2 -adios2_trace" #how to execute tau_exec; the -T arguments should mirror the makefile name <------------ ***SET ME*** |
| 77 | +TAU_PYTHON="tau_python -T papi,mpi,pthread,pdt,adios2 -tau-python-interpreter=python3 -adios2_trace -tau-python-args=-u" #how to execute tau_python. Note that passing -u to python forces it to not buffer stdout so we can pipe it |
| 78 | + #to tee in realtime <--- SET ME (if !python3) |
| 79 | + |
| 80 | +export EXE_NAME=main #the name of the executable (without path) <------------ ***SET ME*** |
| 81 | + |
| 82 | +TAU_ADIOS2_PATH=chimbuko/adios2 #path where the adios2 files are to be stored. Chimbuko services creates the directory chimbuko/adios2 in the working directory and this should be used by default |
| 83 | +TAU_ADIOS2_FILE_PREFIX=tau-metrics #the prefix of tau adios2 files; full filename is ${TAU_ADIOS2_PREFIX}-${EXE_NAME}-${RANK}.bp |
| 84 | + |
| 85 | + |
| 86 | + |
| 87 | + |
| 88 | + |
| 89 | + |
| 90 | + |
| 91 | + |
| 92 | + |
| 93 | + |
| 94 | + |
| 95 | + |
| 96 | + |
| 97 | +########################################################################### |
| 98 | +# NON-USER VARIABLES BELOW = DON'T MODIFY THESE!! |
| 99 | +########################################################################### |
| 100 | +#Extra processing |
| 101 | +export TAU_ADIOS2_FILENAME="${TAU_ADIOS2_PATH}/${TAU_ADIOS2_FILE_PREFIX}" |
| 102 | + |
| 103 | +if [[ ${backend_root} == "infer" ]]; then |
| 104 | + if [[ $(which provdb_admin) == "" ]]; then |
| 105 | + echo "When inferring the backend root directory, could not find provdb_admin in PATH. Please add your Chimbuko bin directory to PATH" |
| 106 | + exit 1 |
| 107 | + fi |
| 108 | + |
| 109 | + backend_root=$( readlink -f $(which provdb_admin | sed 's/provdb_admin//')/../ ) |
| 110 | +fi |
| 111 | + |
| 112 | +if [[ ${chimbuko_services} == "infer" ]]; then |
| 113 | + chimbuko_services="${backend_root}/scripts/launch/run_services.sh" |
| 114 | + if [ ! -f "${chimbuko_services}" ]; then |
| 115 | + echo "Could not infer service script location: service script does not exist at ${chimbuko_services}!" |
| 116 | + exit 1 |
| 117 | + fi |
| 118 | +fi |
| 119 | + |
0 commit comments