First stage of implementing provenance post-pruning. A new ADDataInterface derived class can be used to pass the anomalies back into the AD algorithm on the provDB, eliminating entries that are no longer considered anomalous. A new unit test is included to aid development and checking.

To support the above:
	ADDataInterface can now be default constructed with the number of datasets set after construction
	Added a function to ADOutlier to pass in the initial global model
	Setting ADOutlier::setGlobalModelSyncFrequency to <= 0 will now completely disable updates to the global model
	Added a ProvDBtester to unit_tests/provdb which acts as a standalone, in-memory implementation of the provDB

Author: giltirn

configure.ac

@@ -217,7 +217,7 @@ AC_CONFIG_HEADERS([chimbuko_config.h])
 AC_SUBST([PS_FLAGS])
 
 AC_CONFIG_SRCDIR([src/dummy])
-AC_CONFIG_FILES([Makefile src/Makefile app/Makefile sphinx/Makefile include/Makefile 3rdparty/Makefile test/Makefile test/unit_tests/Makefile test/unit_tests/ad/Makefile test/unit_tests/util/Makefile test/unit_tests/pserver/Makefile test/unit_tests/net/Makefile test/unit_tests/param/Makefile scripts/Makefile scripts/launch/Makefile sim/Makefile sim/src/Makefile sim/main/Makefile sim/include/Makefile])
+AC_CONFIG_FILES([Makefile src/Makefile app/Makefile sphinx/Makefile include/Makefile 3rdparty/Makefile test/Makefile test/unit_tests/Makefile test/unit_tests/ad/Makefile test/unit_tests/util/Makefile test/unit_tests/pserver/Makefile test/unit_tests/net/Makefile test/unit_tests/param/Makefile test/unit_tests/provdb/Makefile scripts/Makefile scripts/launch/Makefile sim/Makefile sim/src/Makefile sim/main/Makefile sim/include/Makefile])
 
 AC_CONFIG_FILES([app/ws_flask_stat.py:app/ws_flask_stat.py app/sst_view_parse.pl:app/sst_view_parse.pl])
 AC_CONFIG_FILES([run_test.sh:run_test.sh test/run_all.sh:test/run_all.sh test/run_ad.sh:test/run_ad.sh test/run_net.sh:test/run_net.sh test/run_provdb_client_test.sh:test/run_provdb_client_test.sh test/run_provdb_autoshutdown_test.sh:test/run_provdb_autoshutdown_test.sh test/unit_tests/run_all.sh:test/unit_tests/run_all.sh test/run_ad_with_provdb.sh:test/run_ad_with_provdb.sh test/run_stat_sender.sh:test/run_stat_sender.sh scripts/launch/run_services.sh:scripts/launch/run_services.sh], [chmod u+x $(echo $ac_tag | sed s/.*\://)])

include/Makefile.am

@@ -1 +1 @@
-nobase_include_HEADERS = chimbuko/modules/performance_analysis/chimbuko.hpp chimbuko/modules/performance_analysis/ad/FuncAnomalyMetrics.hpp chimbuko/modules/performance_analysis/ad/ADLocalAnomalyMetrics.hpp chimbuko/modules/performance_analysis/ad/FuncStats.hpp chimbuko/modules/performance_analysis/ad/ADAnomalyProvenance.hpp chimbuko/modules/performance_analysis/ad/ADParser.hpp chimbuko/modules/performance_analysis/ad/ADMonitoring.hpp chimbuko/modules/performance_analysis/ad/ADNormalEventProvenance.hpp chimbuko/modules/performance_analysis/ad/ADLocalFuncStatistics.hpp chimbuko/modules/performance_analysis/ad/ADMetadataParser.hpp chimbuko/modules/performance_analysis/ad/AnomalyData.hpp chimbuko/modules/performance_analysis/ad/ADDefine.hpp chimbuko/modules/performance_analysis/ad/ADglobalFunctionIndexMap.hpp chimbuko/modules/performance_analysis/ad/ADExecDataInterface.hpp chimbuko/modules/performance_analysis/ad/ADcombinedPSdata.hpp chimbuko/modules/performance_analysis/ad/ADEvent.hpp chimbuko/modules/performance_analysis/ad/ADCounter.hpp chimbuko/modules/performance_analysis/ad/ADLocalCounterStatistics.hpp chimbuko/modules/performance_analysis/ad/ExecData.hpp chimbuko/modules/performance_analysis/provdb/ProvDBmoduleSetup.hpp chimbuko/modules/performance_analysis/module.hpp chimbuko/modules/performance_analysis/pserver/PSmoduleDataManager.hpp chimbuko/modules/performance_analysis/pserver/GlobalAnomalyStats.hpp chimbuko/modules/performance_analysis/pserver/AggregateFuncAnomalyMetrics.hpp chimbuko/modules/performance_analysis/pserver/PSglobalFunctionIndexMap.hpp chimbuko/modules/performance_analysis/pserver/AggregateFuncStats.hpp chimbuko/modules/performance_analysis/pserver/AggregateAnomalyData.hpp chimbuko/modules/performance_analysis/pserver/NetPayloadRecvCombinedADdata.hpp chimbuko/modules/performance_analysis/pserver/PScommon.hpp chimbuko/modules/performance_analysis/pserver/FunctionProfile.hpp chimbuko/modules/performance_analysis/pserver/GlobalAnomalyMetrics.hpp chimbuko/modules/performance_analysis/pserver/AggregateFuncAnomalyMetricsAllRanks.hpp chimbuko/modules/performance_analysis/pserver/GlobalCounterStats.hpp chimbuko/modules/performance_analysis/AD.hpp chimbuko/modules/performance_analysis/pserver.hpp chimbuko/modules/factory.hpp chimbuko/core/chimbuko.hpp chimbuko/core/ad/ADOutlier.hpp chimbuko/core/ad/ADNetClient.hpp chimbuko/core/ad/ADDataInterface.hpp chimbuko/core/ad/ADProvenanceDBclient.hpp chimbuko/core/ad/ADcmdLineArgs.hpp chimbuko/core/ad/utils.hpp chimbuko/core/ad/ADio.hpp chimbuko/core/provdb/ProvDBclient.hpp chimbuko/core/provdb/ProvDBmoduleSetupCore.hpp chimbuko/core/provdb/ProvDBengine.hpp chimbuko/core/provdb/setup.hpp chimbuko/core/verbose.hpp chimbuko/core/pserver/PSProvenanceDBclient.hpp chimbuko/core/pserver/PSmoduleDataManagerCore.hpp chimbuko/core/pserver/PSparamManager.hpp chimbuko/core/pserver/PSstatSender.hpp chimbuko/core/param/sstd_param.hpp chimbuko/core/param/copod_param.hpp chimbuko/core/param/hbos_param.hpp chimbuko/core/message.hpp chimbuko/core/net.hpp chimbuko/core/pserver.hpp chimbuko/core/provdb.hpp chimbuko/core/param.hpp chimbuko/core/net/zmqme_net.hpp chimbuko/core/net/mpi_net.hpp chimbuko/core/net/zmq_net.hpp chimbuko/core/net/local_net.hpp chimbuko/core/util/chunkAllocator.hpp chimbuko/core/util/RunStats.hpp chimbuko/core/util/ADIOS2parseUtils.hpp chimbuko/core/util/json.hpp chimbuko/core/util/environment.hpp chimbuko/core/util/pointerRegistry.hpp chimbuko/core/util/time.hpp chimbuko/core/util/map.hpp chimbuko/core/util/error.hpp chimbuko/core/util/string.hpp chimbuko/core/util/DispatchQueue.hpp chimbuko/core/util/PerfStats.hpp chimbuko/core/util/curlJsonSender.hpp chimbuko/core/util/RunMetric.hpp chimbuko/core/util/memutils.hpp chimbuko/core/util/Histogram.hpp chimbuko/core/util/barrier.hpp chimbuko/core/util/threadPool.hpp chimbuko/core/util/hash.hpp chimbuko/core/util/mtQueue.hpp chimbuko/core/util/serialize.hpp chimbuko/core/util/commandLineParser.hpp 
+nobase_include_HEADERS = chimbuko/modules/performance_analysis/chimbuko.hpp chimbuko/modules/performance_analysis/ad/FuncAnomalyMetrics.hpp chimbuko/modules/performance_analysis/ad/ADLocalAnomalyMetrics.hpp chimbuko/modules/performance_analysis/ad/FuncStats.hpp chimbuko/modules/performance_analysis/ad/ADAnomalyProvenance.hpp chimbuko/modules/performance_analysis/ad/ADParser.hpp chimbuko/modules/performance_analysis/ad/ADMonitoring.hpp chimbuko/modules/performance_analysis/ad/ADNormalEventProvenance.hpp chimbuko/modules/performance_analysis/ad/ADLocalFuncStatistics.hpp chimbuko/modules/performance_analysis/ad/ADMetadataParser.hpp chimbuko/modules/performance_analysis/ad/AnomalyData.hpp chimbuko/modules/performance_analysis/ad/ADDefine.hpp chimbuko/modules/performance_analysis/ad/ADglobalFunctionIndexMap.hpp chimbuko/modules/performance_analysis/ad/ADExecDataInterface.hpp chimbuko/modules/performance_analysis/ad/ADcombinedPSdata.hpp chimbuko/modules/performance_analysis/ad/ADEvent.hpp chimbuko/modules/performance_analysis/ad/ADCounter.hpp chimbuko/modules/performance_analysis/ad/ADLocalCounterStatistics.hpp chimbuko/modules/performance_analysis/ad/ExecData.hpp chimbuko/modules/performance_analysis/provdb/ProvDBpruneOutlierInterface.hpp chimbuko/modules/performance_analysis/provdb/ProvDBmoduleSetup.hpp chimbuko/modules/performance_analysis/module.hpp chimbuko/modules/performance_analysis/pserver/PSmoduleDataManager.hpp chimbuko/modules/performance_analysis/pserver/GlobalAnomalyStats.hpp chimbuko/modules/performance_analysis/pserver/AggregateFuncAnomalyMetrics.hpp chimbuko/modules/performance_analysis/pserver/PSglobalFunctionIndexMap.hpp chimbuko/modules/performance_analysis/pserver/AggregateFuncStats.hpp chimbuko/modules/performance_analysis/pserver/AggregateAnomalyData.hpp chimbuko/modules/performance_analysis/pserver/NetPayloadRecvCombinedADdata.hpp chimbuko/modules/performance_analysis/pserver/PScommon.hpp chimbuko/modules/performance_analysis/pserver/FunctionProfile.hpp chimbuko/modules/performance_analysis/pserver/GlobalAnomalyMetrics.hpp chimbuko/modules/performance_analysis/pserver/AggregateFuncAnomalyMetricsAllRanks.hpp chimbuko/modules/performance_analysis/pserver/GlobalCounterStats.hpp chimbuko/modules/performance_analysis/AD.hpp chimbuko/modules/performance_analysis/pserver.hpp chimbuko/modules/factory.hpp chimbuko/core/chimbuko.hpp chimbuko/core/ad/ADOutlier.hpp chimbuko/core/ad/ADNetClient.hpp chimbuko/core/ad/ADDataInterface.hpp chimbuko/core/ad/ADProvenanceDBclient.hpp chimbuko/core/ad/ADcmdLineArgs.hpp chimbuko/core/ad/utils.hpp chimbuko/core/ad/ADio.hpp chimbuko/core/provdb/ProvDBclient.hpp chimbuko/core/provdb/ProvDBmoduleSetupCore.hpp chimbuko/core/provdb/ProvDBengine.hpp chimbuko/core/provdb/setup.hpp chimbuko/core/verbose.hpp chimbuko/core/pserver/PSProvenanceDBclient.hpp chimbuko/core/pserver/PSmoduleDataManagerCore.hpp chimbuko/core/pserver/PSparamManager.hpp chimbuko/core/pserver/PSstatSender.hpp chimbuko/core/param/sstd_param.hpp chimbuko/core/param/copod_param.hpp chimbuko/core/param/hbos_param.hpp chimbuko/core/message.hpp chimbuko/core/net.hpp chimbuko/core/pserver.hpp chimbuko/core/provdb.hpp chimbuko/core/param.hpp chimbuko/core/net/zmqme_net.hpp chimbuko/core/net/mpi_net.hpp chimbuko/core/net/zmq_net.hpp chimbuko/core/net/local_net.hpp chimbuko/core/util/chunkAllocator.hpp chimbuko/core/util/RunStats.hpp chimbuko/core/util/ADIOS2parseUtils.hpp chimbuko/core/util/json.hpp chimbuko/core/util/environment.hpp chimbuko/core/util/pointerRegistry.hpp chimbuko/core/util/time.hpp chimbuko/core/util/map.hpp chimbuko/core/util/error.hpp chimbuko/core/util/string.hpp chimbuko/core/util/DispatchQueue.hpp chimbuko/core/util/PerfStats.hpp chimbuko/core/util/curlJsonSender.hpp chimbuko/core/util/RunMetric.hpp chimbuko/core/util/memutils.hpp chimbuko/core/util/Histogram.hpp chimbuko/core/util/barrier.hpp chimbuko/core/util/threadPool.hpp chimbuko/core/util/hash.hpp chimbuko/core/util/mtQueue.hpp chimbuko/core/util/serialize.hpp chimbuko/core/util/commandLineParser.hpp 

include/chimbuko/core/ad/ADDataInterface.hpp

@@ -62,6 +62,12 @@ namespace chimbuko {
     };
 
     ADDataInterface(size_t ndataset): m_dset_anom(ndataset){}
+    ADDataInterface(){}
+    
+    /**
+     * @brief Set the number of datasets, designed for use if instance was default-constructed
+     */
+    inline void setNdataSets(size_t ndataset){ m_dset_anom.resize(ndataset); }
 
     /**
      * @brief Return the number of data sets

include/chimbuko/core/ad/ADOutlier.hpp

@@ -85,7 +85,14 @@ namespace chimbuko {
     ParamInterface const* get_global_parameters() const{ return m_param; }
 
     /**
-     * @brief Set how often (in steps, or equivalently calls to "run") the global model is updated
+     * @brief Set the global parameters, overwriting the existing global model. Here the input is in serialized form
+     *
+     * Use in conjunction with setGlobalModelSyncFrequency(0) to set and freeze the model, not allowing it to be modified by the data
+     */
+    void setGlobalParameters(const std::string &to);
+
+    /**
+     * @brief Set how often (in steps, or equivalently calls to "run") the global model is updated. If  to <= 0, the global model will never be updated
      */
     void setGlobalModelSyncFrequency(int to){ m_global_model_sync_freq = to; }
 
@@ -114,7 +121,7 @@ namespace chimbuko {
     int m_sync_call_count;                   /**< count of calls to sync_param */
     int m_global_model_sync_freq;            /**< how often the local model is pushed and synchronized with the globel model (default 1)*/
 
-    int m_rank;                              /**< rank index*/
+    int m_rank;                              /**< rank index, used only for staggering pserver sync events*/
     PerfStats *m_perf;
   };
 

include/chimbuko/core/param.hpp

@@ -31,18 +31,18 @@ namespace chimbuko {
     virtual void clear() = 0;
 
     /**
-     * @brief Get the number of functions for which statistics are being collected
+     * @brief Get the number of models for which statistics are being collected
     */
     virtual size_t size() const = 0;
 
     /**
-     * @brief Convert internal run statistics to string format for IO
+     * @brief Convert internal models to string format for IO
      * @return Run statistics in string format
      */
     virtual std::string serialize() const = 0;
 
     /**
-     * @brief Update the internal run statistics with those included in the serialized input map
+     * @brief Update the internal models with those included in the serialized input map
      * @param parameters The parameters in serialized format
      * @param return_update Indicates that the function should return a serialized copy of the updated parameters
      * @return An empty string or a serialized copy of the updated parameters depending on return_update
@@ -69,8 +69,8 @@ namespace chimbuko {
     virtual void update(const std::vector<ParamInterface*> &other);
 
     /**
-     * @brief Set the internal run statistics to match those included in the serialized input map. Overwrite performed only for those keys in input.
-     * @param runstats The serialized input map
+     * @brief Set the internal run statistics to match those included in the serialized input model. Overwrite performed only for those model indices in te input.
+     * @param parameters The serialized input model
      */
     virtual void assign(const std::string& parameters) = 0;
 

include/chimbuko/core/util/RunStats.hpp

@@ -169,11 +169,11 @@ namespace chimbuko {
     bool equiv(const RunStats &b) const;
 
     /**
-     * @brief Set the eta parameter
+     * @brief Set the eta (mean) parameter
      */
     void set_eta(double to){ m_eta = to; }
     /**
-     * @brief Set the rho parameter
+     * @brief Set the rho parameter (variance * [count-1])
      */
     void set_rho(double to){ m_rho = to; }
     /**

include/chimbuko/modules/performance_analysis/provdb/ProvDBpruneOutlierInterface.hpp

@@ -0,0 +1,52 @@
+#pragma once
+#include <chimbuko_config.h>
+#include <chimbuko/core/ad/ADDataInterface.hpp>
+#include <chimbuko/core/ad/ADOutlier.hpp>
+
+#include<string>
+#include <sonata/Database.hpp>
+
+namespace chimbuko {
+  namespace modules{
+    namespace performance_analysis{
+
+      /**
+       * @brief The interface class between the provDB data and the AD algorithm
+       */
+      class ProvDBpruneOutlierInterface: public ADDataInterface{
+      public:
+	ProvDBpruneOutlierInterface(sonata::Database &db);
+
+	/**
+	 * @brief Get the values associated with each recorded anomaly
+	 */
+	std::vector<Elem> getDataSet(size_t dset_index) const override;
+	
+	/**
+	 * @brief Check the newly assigned label is still anomaly, otherwise erase the element from the database
+	 */
+	void recordDataSetLabelsInternal(const std::vector<Elem> &data, size_t dset_index) override;
+
+	
+	/**
+	 * @brief Return the function index (a unique index associated with a program index/function name combination) associated with a given dataset index
+	 */
+	size_t getDataSetModelIndex(size_t dset_index) const;
+
+
+      private:
+	sonata::Database &m_database;
+	std::unique_ptr<sonata::Collection> m_collection;
+	std::unordered_map<unsigned long, std::vector<std::pair<uint64_t, double> > > m_data; //[fid] -> [  (record_id, value), ... ]
+      };
+
+
+      /**
+       * @brief Instantiate the AD algorithm with the provided parameters and use it to prune entries from the database that are no longer outliers
+       */
+      void ProvDBpruneOutliers(const std::string &algorithm, const ADOutlier::AlgoParams &algo_params, const std::string &params_ser, sonata::Database &db);
+      
+
+    }
+  }
+}