Merge branch 'sm_release' into ckelly_develop

Author: giltirn

docs/_sources/introduction/ad.rst.txt

@@ -57,8 +57,12 @@ the more anomalies or the more false positives).
 
 Advanced anomaly analysis
 ~~~~~~~~~~~~~~~~~~~~~~~~~
-TBD
+A determistic and non-parametric statistical anomaly detection algorithm called Histogram Based Outilier Scoring (HBOS) is implemented as part of Chimbuko's anomaly analysis module. HBOS is an unsupervised anomaly detection algorithm which scores data in linear time. It supports dynamic bin widths which ensures long-tail distributions of function executions are captured and global anomalies are detected better. HBOS normalizes the histogram and calculates the anomaly scores by taking inverse of estimated densities of function executions. The score is a multiplication of the inverse of the estimated densities given by the following Equation
 
+.. math::
+    HBOS_score_{i} = log2 (1 / exec_density_{i})
+
+where :math:`i` is a function execution and :math:`exec_density_{i}` is function execution probability. HBOS works in :math:`O(nlogn)` using dynamic bin-width or in linear time :math:`O(n)` using fixed bin width. After scoring, the top 1% of scores are filtered as anomalous function executions. This filter value can be set at runtime to adjust the density of detected anomalies. 
 
 Stream local viz data
 ---------------------

include/chimbuko/ad/ADOutlier.hpp

@@ -114,7 +114,7 @@ namespace chimbuko {
     int m_rank;                              /**< this process rank                      */
     bool m_use_ps;                           /**< true if the parameter server is in use */
     ADThreadNetClient* m_net_client;                 /**< interface for communicating to parameter server */
-    
+
     std::unordered_map< std::array<unsigned long, 4>, size_t, ArrayHasher<unsigned long,4> > m_local_func_exec_count; /**< Map(program id, rank id, thread id, func id) -> number of times encountered on this node*/
 
     const ExecDataMap_t * m_execDataMap;     /**< execution data map */
@@ -192,11 +192,13 @@ namespace chimbuko {
    */
   class ADOutlierHBOS : public ADOutlier {
   public:
+
     /**
      * @brief Construct a new ADOutlierHBOS object
      *
      */
     ADOutlierHBOS(OutlierStatistic stat = ExclusiveRuntime, double threshold = 0.99, bool use_global_threshold = true);
+
     /**
      * @brief Destroy the ADOutlierHBOS object
      *
@@ -238,16 +240,19 @@ namespace chimbuko {
     std::pair<size_t, size_t> sync_param(ParamInterface const* param) override;
 
     /**
-     * scott's rule for bin_width estimation
+     * @brief Scott's rule for bin_width estimation during histogram formation
      */
     double _scott_binWidth(std::vector<double>& vals);
 
+    /**
+     * @brief Assigns samples to corresponding bins in Histogram. Similar to numpy digitize in Python
+     */
     int np_digitize_get_bin_inds(const double& X, const std::vector<double>& bin_edges);
 
   private:
-    double m_alpha; /**< alpha */
+    double m_alpha; /**< Used to prevent log2 overflow */
     double m_threshold; /**< Threshold used to filter anomalies in HBOS*/
-    bool m_use_global_threshold; /**< Flas to use global threshold*/
+    bool m_use_global_threshold; /**< Flag to use global threshold*/
     //double m_threshold; /** sync with global threshold */
     OutlierStatistic m_statistic; /** Which statistic to use for outlier detection */
 

include/chimbuko/param/hbos_param.hpp

@@ -15,18 +15,43 @@ namespace chimbuko {
   class Histogram {
 
   public:
+
+    /**
+     * @brief Construct a Histogram object
+     */
     Histogram();
+
+    /**
+     * @brief Destroy Histogram object
+     */
     ~Histogram();
 
+    /**
+     * @brief Data structure that stores Histogram data ( bin counts, bin edges)
+     */
     struct Data {
 
       double glob_threshold; /**< global threshold used to filter anomalies*/
       std::vector<int> counts; /**< Bin counts in Histogram*/
       std::vector<double> bin_edges; /**< Bin edges in Histogram*/
 
+      /**
+       * @brief Initialize histogram data
+       */
       Data(){
+
+        /**
+         * @brief Resets histogram data during initialization
+         */
         clear();
       }
+
+      /**
+       * @brief Initialize histogram data with existing histogram data
+       * @param g_threshold: Global Threshold
+       * @param h_counts: a vector<int> of histogram bin counts
+       * @param h_bin_edges: a vector<double> of histogram bin edges
+       */
       Data(const double& g_threshold, const std::vector<int>& h_counts, const std::vector<double>& h_bin_edges ) {
         glob_threshold = g_threshold;
         counts = h_counts;
@@ -56,82 +81,111 @@ namespace chimbuko {
 
     void push (double x);
 
+    /**
+     * @brief returns reference to current histogram Data
+     * @return Data: Histogram data (bin counts, bin edges)
+     */
     const Data &get_histogram() const{ return m_histogram; }
 
     /**
      * @brief Set the internal variables from an instance of Histogram Data
+     * @param d: Histogram Data (bin counts, bin edges)
      */
     void set_hist_data(const Data& d);
 
     /**
      * @brief Create an instance of this class from a Histogram Data instance
+     * @param d: Histogram Data (bin counts, bin edges)
+     * @return Instance of Histogram
      */
     static Histogram from_hist_data(const Data& d) {
       Histogram histdata;
       histdata.set_hist_data(d);
       return histdata;
     }
+
     /**
      * @brief Create new histogram locally for AD module's batch data instances
+     * @param r_times: a vector<double> of function run times
+     * @return returns 0 if success, else -1
      */
     int create_histogram(const std::vector<double>& r_times);
 
-
+    /**
+     * @brief merges a Histogram with function runtimes
+     * @param g: Histogram to merge
+     * @param runtimes: Function runtimes
+     * @return 0 if successful, -1 if failed
+     */
     int merge_histograms(const Histogram& g, const std::vector<double>& runtimes);
 
     /**
-     * @brief Combine two Histogram instances such that the resulting statistics are the union of the two
+     * @brief Combine two Histogram instances such that the resulting statistics are the union of the two Histograms
+     * @param g: Histogram to merge into
+     * @param l: Histogram to merge
+     * @return result: Merged Histogram
      */
     friend Histogram operator+(const Histogram& g, const Histogram& l);
 
     /**
-     * @brief Combine two Histogram instances such that the resulting statistics are the union of the two
+     * @brief Combine two Histogram instances such that the resulting statistics are the union of the two Histograms
+     * @param h: Histogram to merge
+     * @return result: Merged Histogram
      */
     Histogram & operator+=(const Histogram& h);
 
 
     /**
-     * @brief setd global threshold for anomaly filtering
+     * @brief set global threshold for anomaly filtering
      */
     void set_glob_threshold(const double& l) { m_histogram.glob_threshold = l;}
 
     /*
      * @brief set bin counts in Histogram
+     * @param c: vector of bin counts
      */
     void set_counts(const std::vector<int> & c) { m_histogram.counts = c; }
 
     /*
      * @brief set bin edges in Histogram
+     * @param be: vector of bin edges
      */
     void set_bin_edges(const std::vector<double>& be) {m_histogram.bin_edges = be;}
 
     /*
-     * @brief Update counts in Histogram
+     * @brief New bin counts in Histogram
+     * @param count: bin count value
      */
     void add2counts(const int& count) {m_histogram.counts.push_back(count);}
 
     /*
      * @brief Update counts for a given index of bin in histogram
+     * @param id: index of bin in Histogram
+     * @param count: bin count value to update
      */
     void add2counts(const int& id, const int& count) {m_histogram.counts[id] += count;}
 
     /*
-     * @brief Update bin edges in histogram
+     * @brief New bin edges in histogram
+     * @param bin_edge: vector of bin edges of histogram
      */
     void add2binedges(const double& bin_edge) {m_histogram.bin_edges.push_back(bin_edge);}
 
     /*
-     * @brief get global threshold from global histogram
+     * @brief get current value of global threshold from global histogram
+     * @return global threshold
      */
     const double& get_threshold() const {return m_histogram.glob_threshold;}
 
     /*
-     * @brief Get bin counts of Histogram
+     * @brief Get current vector of bin counts of Histogram
+     * @return vector of bin counts
      */
     const std::vector<int>& counts() const {return m_histogram.counts;}
 
     /*
-     * @brief Get bin edges of histogram
+     * @brief Get current vector of bin edges of histogram
+     * @return vector of bin edges
      */
     const std::vector<double>& bin_edges() const {return m_histogram.bin_edges;}
 
@@ -141,11 +195,12 @@ namespace chimbuko {
     nlohmann::json get_json() const;
 
   private:
-    Data m_histogram; /**< Histogram Data*/
+    Data m_histogram; /**< Histogram Data (bin counts, bin edges)*/
 
     /*
      * @brief Compute bin width based on Scott's rule
      * @param vals: vector of runtimes
+     * @return computed bin width
      */
     static double _scott_binWidth(const std::vector<double> & vals);
 
@@ -155,10 +210,13 @@ namespace chimbuko {
      * @param global_edges: bin edges in global histogram on pserver
      * @param local_counts: bin counts in local histogram in AD module
      * @param local_edges: bin edges in local histogram in AD module
+     * @return computed bin width
      */
     static double _scott_binWidth(const std::vector<int> & global_counts, const std::vector<double> & global_edges, const std::vector<int> & local_counts, const std::vector<double> & local_edges);
 
   };
+
+
   Histogram operator+(const Histogram& g, const Histogram& l);
 
   /**
@@ -173,6 +231,7 @@ namespace chimbuko {
      */
     void clear() override;
 
+
     const int find(const unsigned long& func_id);
 
     /**
@@ -263,7 +322,7 @@ namespace chimbuko {
     nlohmann::json get_algorithm_params(const unsigned long func_id) const override;
 
   private:
-    std::unordered_map<unsigned long, Histogram> m_hbosstats;
+    std::unordered_map<unsigned long, Histogram> m_hbosstats; /**< Map of func_id and corresponding Histogram*/
   };
 
 

sphinx/source/api/api_code.rst

@@ -92,7 +92,7 @@ ADNormalEventProvenance
 .. doxygenfile:: ADNormalEventProvenance.hpp
 	:project: api
 	:path: ../../include/chimbuko/ad//ADNormalEventProvenance.hp
-	       
+
 ADOutlier
 ---------
 
@@ -128,7 +128,7 @@ AnomalyData
 	:project: api
 	:path: ../../include/chimbuko/ad//AnomalyData.hpp
 
-	       
+
 ExecData
 --------
 
@@ -156,6 +156,13 @@ ParamInterface
    :project: api
    :path: ../../../include/chimbuko/param.hpp
 
+HbosParam
+---------
+
+.. doxygenfile:: hbos_param.hpp
+   :project: api
+   :path: ../../../include/chimbuko/param/hbos_param.hpp
+
 SstdParam
 ---------
 
@@ -204,15 +211,15 @@ PSProvenanceDBclient
 .. doxygenfile:: PSProvenanceDBclient.hpp
 	:project: api
 	:path: ../../include/chimbuko/pserver/PSProvenanceDBclient.hpp
-	       
+
 PSstatSender
 ------------
 
 .. doxygenfile:: PSstatSender.hpp
 	:project: api
 	:path: ../../include/chimbuko/pserver/PSstatSender.hpp
 
-	  
+
 Network
 ~~~~~~~
 
@@ -246,7 +253,7 @@ ZMQMENet
 	:project: api
 	:path: ../../include/chimbuko/net/zmqme_net.hpp
 
-	  
+
 Message
 ~~~~~~~
 
@@ -300,7 +307,7 @@ error
 .. doxygenfile:: error.hpp
 	:project: api
 	:path: ../../include/chimbuko/util//error.hpp
-	       
+
 hash
 ----
 
@@ -321,7 +328,7 @@ memutils
 .. doxygenfile:: memutils.hpp
 	:project: api
 	:path: ../../include/chimbuko/util//memutils.hpp
-	       
+
 mtQueue
 -------
 
@@ -370,7 +377,7 @@ time
 .. doxygenfile:: time.hpp
 	:project: api
 	:path: ../../include/chimbuko/util//time.hpp
-	       
+
 verbose
 -------
 

sphinx/source/appendix/appendix.rst

@@ -0,0 +1,6 @@
+
+This is dedicated to Installation and Usage section. Description of various parameters used during Chimbuko's Install and Usage are described here.
+
+.. include:: appendix_install.rst
+.. include:: appendix_instrument_with_tau.rst
+.. include:: appendix_usage.rst

sphinx/source/appendix/appendix_install.rst

@@ -0,0 +1,33 @@
+*************
+Installation
+*************
+
+Gtest Installation
+~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+   apt-get install libgtest-dev
+   cd /usr/src/gtest
+   cmake CMakelists.txt
+   make
+   cp *.a /usr/lib
+
+ADIOS2 Build Location
+~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+   ADIOS2_INSTALL_DIR=/gpfs/alpine/world-shared/csc143/jyc/summit/sw/adios2/devel/gcc
+
+GoogleTest on Summit
+~~~~~~~~~~~~~~~~~~~~
+
+.. code:: bash
+
+   cd ${GTEST_SOURCE_DIR}
+   wget https://github.com/google/googletest/archive/release-1.10.0.tar.gz
+   tar -xvzf release-1.10.0.tar.gz
+   cd ${GTEST_BUILD_DIR}
+   CC=gcc CXX=g++ cmake -DCMAKE_INSTALL_PREFIX=${GTEST_INSTALL_DIR} ${GTEST_SOURCE_DIR}/googletest-release-1.10.0
+   make install