Make StatSummarizer more useful.

Change: 121702743

1 files changed, 99 insertions, 57 deletions

diff --git a/tensorflow/core/util/stat_summarizer.h b/tensorflow/core/util/stat_summarizer.h
index 7c351d3b09..1a6334ee1a 100644
--- a/tensorflow/core/util/stat_summarizer.h
+++ b/tensorflow/core/util/stat_summarizer.h
@@ -18,12 +18,12 @@ limitations under the License.
 
 #include <cmath>
 #include <limits>
-#include <list>
 #include <map>
 #include <sstream>
 #include <string>
 
 #include <stdlib.h>
+
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/platform/types.h"
@@ -32,60 +32,73 @@ namespace tensorflow {
 
 class GraphDef;
 class StepStats;
+class NodeExecStats;
 
-static const int kNumValues = 1000;
-
-template <typename T>
+template <typename ValueType, typename HighPrecisionValueType = double>
 class Stat {
  public:
-  Stat<T>() { Reset(); }
+  void UpdateStat(ValueType v) {
+    newest_ = v;
+    max_ = std::max(v, max_);
+    min_ = std::min(v, min_);
+    ++count_;
+    sum_ += v;
+    squared_sum_ += static_cast<HighPrecisionValueType>(v) * v;
+  }
 
-  void Reset() {
-    values_.clear();
-    min_value_ = std::numeric_limits<T>::max();
-    max_value_ = std::numeric_limits<T>::min();
-    avg_value_ = 0;
-    current_value_ = 0;
-    std_deviation_ = 0;
+  void Reset() { new (this) Stat<ValueType, HighPrecisionValueType>(); }
+
+  bool empty() const { return count_ == 0; }
+
+  ValueType newest() const { return newest_; }
+
+  ValueType max() const { return max_; }
+
+  ValueType min() const { return min_; }
+
+  int64 count() const { return count_; }
+
+  ValueType sum() const { return sum_; }
+
+  HighPrecisionValueType squared_sum() const { return squared_sum_; }
+
+  bool all_same() const { return (count_ == 0 || min_ == max_); }
+
+  HighPrecisionValueType avg() const {
+    return empty() ? std::numeric_limits<ValueType>::quiet_NaN()
+                   : static_cast<HighPrecisionValueType>(sum_) / count_;
   }
 
-  std::list<T> values_;
-  T min_value_;
-  T max_value_;
-  double avg_value_;
-  T current_value_;
-  double std_deviation_;
-
-  void UpdateStat(const T val) {
-    current_value_ = val;
-    values_.push_front(val);
-    while (values_.size() > kNumValues) {
-      values_.pop_back();
-    }
+  ValueType rms() const { return sqrt(squared_sum_ / count_); }
 
-    T total = 0;
-    for (const T curr_val : values_) {
-      min_value_ = std::min(min_value_, curr_val);
-      max_value_ = std::max(max_value_, curr_val);
-      total += curr_val;
-    }
-    avg_value_ = static_cast<double>(total) / values_.size();
+  ValueType std_deviation() const { return all_same() ? 0 : rms() - avg(); }
 
-    double sqr_total = 0.0;
-    for (const T curr_val : values_) {
-      const double delta = avg_value_ - curr_val;
-      sqr_total += delta * delta;
+  void OutputToStream(std::ostream* stream) const {
+    if (empty()) {
+      *stream << "count=0";
+    } else if (all_same()) {
+      *stream << "curr=" << newest_ << " count=" << count_;
+      if (count_ > 1) *stream << "(all same)";
+    } else {
+      *stream << "curr=" << newest_ << " count=" << count_ << " min=" << min_
+              << " max=" << max_ << " avg=" << avg()
+              << " std=" << std_deviation();
     }
-    std_deviation_ = std::sqrt(sqr_total / values_.size());
   }
 
-  friend std::ostream& operator<<(std::ostream& stream, const Stat<T>& stat) {
-    stream << "curr=" << stat.current_value_ << " min=" << stat.min_value_
-           << " max=" << stat.max_value_
-           << " avg=" << static_cast<int64>(stat.avg_value_)
-           << " stddev=" << static_cast<int64>(stat.std_deviation_);
+  friend std::ostream& operator<<(std::ostream& stream,
+                                  const Stat<ValueType>& stat) {
+    stat.OutputToStream(&stream);
     return stream;
   }
+
+ private:
+  ValueType newest_ = 0;
+  ValueType max_ = std::numeric_limits<ValueType>::min();
+  ValueType min_ = std::numeric_limits<ValueType>::max();
+  int64 count_ = 0;
+  ValueType sum_ = 0;
+  HighPrecisionValueType squared_sum_ = 0;
 };
 
 // A class intended to make performance analysis easier by collecting StepStats
@@ -100,28 +113,57 @@ class StatSummarizer {
 
   // Prints all the accumulated runtime stats in a tab-separated format which
   // can be pasted into a spreadsheet for further analysis.
-  void PrintStepStats();
+  void PrintStepStats() const;
 
-  void Reset() {
-    num_runs_ = 0;
+  // Summarizes all nodes' stat in the order of node names defined in the graph.
+  std::string GetStatsByOrderOfNodeDefinitions() const;
+
+  // Summarizes all nodes' stat in the order of nodes getting executed.
+  std::string GetStatsByRunOrder() const;
 
-    run_total_us_.Reset();
+  // Summarizes all nodes' stat in the order of top durations.
+  // Will stop printing if either cdf_cutoff_ratio or num_max_nodes_to_print
+  // is hit.
+  std::string GetStatsByTopDurations(
+      double cdf_cutoff_ratio = 1.0,
+      int num_max_nodes_to_print = std::numeric_limits<int>::max()) const;
 
-    timing_total_us_ = 0;
-    timing_totals_.clear();
+  void Reset() {
+    run_total_micros_.Reset();
+    timing_details_.clear();
   }
 
- private:
-  void PrintHeaders();
-  void PrintColumns(const char* name, const char* op, const double time_ms,
-                    const double percentage);
+  // Returns number of runs.
+  int num_runs() const { return run_total_micros_.count(); }
 
-  Stat<int64> run_total_us_;
+  // Returns stats of total microseconds spent by all nodes in each run.
+  const Stat<int64>& run_total_us() const { return run_total_micros_; }
 
-  int num_runs_ = 0;
-  int64 timing_total_us_ = 0;
-  std::vector<string> nodes_;
-  std::map<string, int64> timing_totals_;
+ private:
+  struct Detail {
+    int64 first_start_micros;
+    int64 first_rel_end_micros;
+    int64 total_micros;
+  };
+
+  enum struct SortingMetric {
+    BY_TOTAL_DURATION,
+    BY_RUN_ORDER,
+  };
+
+  std::string GetStatsBySorting(SortingMetric sorting_metric,
+                                double cdf_cutoff_ratio,
+                                int num_max_nodes_to_print) const;
+
+  std::string HeaderString() const;
+  std::string ColumnString(const std::string& name, const Detail& detail,
+                           int64 cumulative_time_us_on_node) const;
+  std::string ShortSummary() const;
+
+  int64 first_node_start_micros_;
+  Stat<int64> run_total_micros_;
+  std::vector<string> nodes_in_def_order_;
+  std::map<std::string, Detail> timing_details_;
   std::map<string, string> node_types_;
 };