Added a prototype 'simulator' for Chimbuko, providing an interface whereby users can manually provide data to the Chimbuko stack

and produce appropriately processed output

Author: giltirn

Makefile.am

@@ -1,3 +1,3 @@
-SUBDIRS = include src app test sphinx 3rdparty scripts
+SUBDIRS = include src app test sphinx 3rdparty scripts sim
 nobase_include_HEADERS = chimbuko_config.h
 

configure.ac

@@ -156,7 +156,7 @@ AC_CONFIG_HEADERS([chimbuko_config.h])
 AC_SUBST([PS_FLAGS])
 
 AC_CONFIG_SRCDIR([src/chimbuko.cpp])
-AC_CONFIG_FILES([Makefile src/Makefile app/Makefile sphinx/Makefile include/Makefile 3rdparty/Makefile test/Makefile test/unit_tests/Makefile test/unit_tests/ad/Makefile test/unit_tests/util/Makefile test/unit_tests/pserver/Makefile test/unit_tests/net/Makefile test/unit_tests/param/Makefile scripts/Makefile scripts/launch/Makefile])
+AC_CONFIG_FILES([Makefile src/Makefile app/Makefile sphinx/Makefile include/Makefile 3rdparty/Makefile test/Makefile test/unit_tests/Makefile test/unit_tests/ad/Makefile test/unit_tests/util/Makefile test/unit_tests/pserver/Makefile test/unit_tests/net/Makefile test/unit_tests/param/Makefile scripts/Makefile scripts/launch/Makefile sim/Makefile sim/src/Makefile sim/main/Makefile])
 
 AC_CONFIG_FILES([app/ws_flask_stat.py:app/ws_flask_stat.py app/sst_view_parse.pl:app/sst_view_parse.pl])
 AC_CONFIG_FILES([run_test.sh:run_test.sh test/run_all.sh:test/run_all.sh test/run_ad.sh:test/run_ad.sh test/run_net.sh:test/run_net.sh test/run_provdb_client_test.sh:test/run_provdb_client_test.sh test/run_provdb_autoshutdown_test.sh:test/run_provdb_autoshutdown_test.sh test/unit_tests/run_all.sh:test/unit_tests/run_all.sh test/run_ad_with_provdb.sh:test/run_ad_with_provdb.sh test/run_stat_sender.sh:test/run_stat_sender.sh scripts/launch/run_services.sh:scripts/launch/run_services.sh], [chmod u+x $(echo $ac_tag | sed s/.*\://)])

sim/Makefile.am

@@ -0,0 +1 @@
+SUBDIRS = src main

sim/include/sim.hpp

@@ -0,0 +1,9 @@
+#pragma once
+
+#include "sim/provdb.hpp"
+#include "sim/id_map.hpp"
+#include "sim/ad_params.hpp"
+#include "sim/pserver.hpp"
+#include "sim/event_id_map.hpp"
+#include "sim/ad.hpp"
+#include "sim/thread_execution_api.hpp"

sim/include/sim/ad.hpp

@@ -0,0 +1,78 @@
+#pragma once
+
+#include<chimbuko/ad/ADProvenanceDBclient.hpp>
+#include<chimbuko/ad/ADMetadataParser.hpp>
+#include<chimbuko/ad/ADNormalEventProvenance.hpp>
+#include<chimbuko/ad/ADEvent.hpp>
+#include<chimbuko/ad/ADCounter.hpp>
+
+namespace chimbuko_sim{
+  using namespace chimbuko;
+
+  enum class CommType { Send, Recv };
+
+  //An object that represents a rank of the AD
+  class ADsim{
+    std::unordered_map<unsigned long, std::list<ExecData_t> > m_all_execs; //map of thread to execs, never flushed
+    std::unordered_map<eventID, CallListIterator_t> m_eventid_map; //map from event index to iterator ; never flushed
+    std::list<CallListIterator_t> m_step_exec_its; //iterators to events on this io step; flushed at end of step
+    std::unordered_map<unsigned long, std::vector<CallListIterator_t>> m_step_func_exec_its; //map of function id to iterators on this io step; flushed at end of step
+
+    int m_step;
+    int m_window_size;
+    int m_pid;
+    int m_rid;
+    unsigned long m_step_start_time;
+    ADNormalEventProvenance m_normal_events;
+    ADCounter m_counters;
+    ADMetadataParser m_metadata;
+    std::unique_ptr<ADProvenanceDBclient> m_pdb_client;
+  public:
+    void init(int window_size, int pid, int rid);
+
+    ADsim(int window_size, int pid, int rid): ADsim(){
+      init(window_size, pid, rid);
+    }
+    ADsim(){}
+
+    ADProvenanceDBclient &getProvDBclient(){ return *m_pdb_client; }
+
+    //Add a function execution on a specific thread
+    CallListIterator_t addExec(const int thread,
+			       const std::string &func_name,
+			       unsigned long start,
+			       unsigned long runtime,
+			       bool is_anomaly,
+			       double outlier_score = 0.);
+
+    //Attach a counter to an execution t_delta us after the start of the execution
+    void attachCounter(const std::string &counter_name,
+		       unsigned long value,
+		       long t_delta,
+		       CallListIterator_t to);
+
+    //Attach a communication event to an execution t_delta us after the start of the execution
+    //partner_rank is the origin rank of a receive or the destination rank of a send
+    void attachComm(CommType type,
+		    unsigned long partner_rank,
+		    unsigned long bytes,
+		    long t_delta,
+		    CallListIterator_t to);
+   
+    //Register a thread index as corresponding to a GPU thread, allowing population of GPU information in provenance data
+    void registerGPUthread(const int tid);
+    
+    //Register a GPU kernel event as originating from a cpu event
+    void bindCPUparentGPUkernel(CallListIterator_t cpu_parent, CallListIterator_t gpu_kern);
+
+    void beginStep(const unsigned long step_start_time);
+
+    void endStep(const unsigned long step_end_time);
+  };
+
+  inline std::ostream & operator<<(std::ostream &os, const CallListIterator_t it){
+    os << it->get_json(true, true).dump(4);
+    return os;
+  }
+
+};

sim/include/sim/ad_params.hpp

@@ -0,0 +1,11 @@
+#include<chimbuko/param/sstd_param.hpp>
+#include "id_map.hpp"
+
+namespace chimbuko_sim{
+  using namespace chimbuko;
+
+  inline SstdParam & globalParams(){ static SstdParam p; return p; }
+  
+  //"register" a function, generating fake statistics given a provided mean and standard deviation
+  void registerFunc(const std::string &func_name, unsigned long mean_runtime, unsigned long std_dev_runtime, int seen_count);
+};

sim/include/sim/event_id_map.hpp

@@ -0,0 +1,22 @@
+#include<chimbuko/ad/ADEvent.hpp>
+
+namespace chimbuko_sim{
+  using namespace chimbuko;
+
+  /**
+   * @brief An implementation of ADEventIDmap interface for the simulator using external maps
+   */
+  class eventIDmap: public ADEventIDmap{
+    const std::unordered_map<unsigned long, std::list<ExecData_t> > &m_thread_exec_map;
+    const std::unordered_map<eventID, CallListIterator_t> &m_eventid_map;
+
+  public:
+    eventIDmap(const std::unordered_map<unsigned long, std::list<ExecData_t> > &thread_exec_map, 
+	       const std::unordered_map<eventID, CallListIterator_t> &eventid_map): m_thread_exec_map(thread_exec_map), m_eventid_map(eventid_map){}
+  
+    CallListIterator_t getCallData(const eventID &event_id) const override;
+
+    std::pair<CallListIterator_t, CallListIterator_t> getCallWindowStartEnd(const eventID &event_id, const int win_size) const override;
+  };
+
+};

sim/include/sim/id_map.hpp

@@ -0,0 +1,25 @@
+#pragma once
+
+#include<string>
+#include<unordered_map>
+
+namespace chimbuko_sim{
+  //All functions must be registered
+  std::unordered_map<std::string, unsigned long> & funcIdxMap();
+
+  //Automatically register counter names with indices
+  class CounterIdxManager{
+    std::unordered_map<std::string, unsigned long> cid_map;
+    std::unordered_map<int, std::string> cname_map;
+
+  public:  
+    /**
+     * @brief Get an index for the counter 'cname'. A new index will be generated the first time a counter is seen
+     */
+    long getIndex(const std::string &cname);
+  
+    const std::unordered_map<int, std::string>* getCounterMap(){ return &cname_map; }
+  };
+
+  inline CounterIdxManager & getCidxManager(){ static CounterIdxManager cman; return cman; }
+};

sim/include/sim/provdb.hpp

@@ -0,0 +1,33 @@
+#include <sonata/Admin.hpp>
+#include <sonata/Provider.hpp>
+#include<chimbuko/ad/ADProvenanceDBclient.hpp>
+
+namespace chimbuko_sim{
+  using namespace chimbuko;
+
+  //A class the acts as the provenance database admin
+  class provDBsim{
+  private:
+    sonata::Provider *provider;
+    sonata::Admin *admin;
+    sonata::Client *client;
+    std::string addr;
+    std::vector<std::string> db_shard_names;
+    std::vector<sonata::Database> db;
+    sonata::Database glob_db;
+  public:
+    provDBsim(int nshards);
+
+    const std::string &getAddr() const{ return addr; }
+  
+    int getNshards() const{ return db.size(); }
+
+    ~provDBsim();
+  };
+
+  //Set before first ADsim is created
+  inline int & nshards(){ static int n=1; return n; }
+
+  inline provDBsim & getProvDB(){ static provDBsim pdb(nshards()); return pdb; }
+
+};

sim/include/sim/pserver.hpp

@@ -0,0 +1,34 @@
+#pragma once
+
+#include<chimbuko/pserver/global_anomaly_stats.hpp>
+#include<chimbuko/pserver/global_counter_stats.hpp>
+
+namespace chimbuko_sim{
+  using namespace chimbuko;
+
+  //An object that represents the pserver for the purposes of aggregating data over AD instances and writing streaming output to disk
+  class pserverSim{
+    GlobalAnomalyStats global_func_stats; //global anomaly statistics
+    GlobalCounterStats global_counter_stats; //global counter statistics
+    PSstatSenderGlobalAnomalyStatsPayload anomaly_stats_payload;
+    PSstatSenderGlobalCounterStatsPayload counter_stats_payload;
+  public:
+
+    pserverSim(): anomaly_stats_payload(&global_func_stats), counter_stats_payload(&global_counter_stats){}
+  
+    //Stand-in for the func statistics comms from AD -> pserver
+    void addAnomalyData(const ADLocalFuncStatistics &loc){
+      global_func_stats.add_anomaly_data(loc);
+    }
+    void addCounterData(const ADLocalCounterStatistics &loc){
+      global_counter_stats.add_counter_data(loc);
+    }
+
+    //Mirror write functionality of PSstatSender
+    void writeStreamingOutput() const;
+  };  
+
+
+  inline pserverSim & getPserver(){ static pserverSim ps; return ps; }
+
+};