Merge pull request #13575 from vjpai/dehist

Move histogram to test/core/util

4 files changed, 472 insertions, 5 deletions

diff --git a/test/core/util/BUILD b/test/core/util/BUILD
index 6443553466..268547f6c9 100644
--- a/test/core/util/BUILD
+++ b/test/core/util/BUILD
@@ -16,7 +16,10 @@ load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_c
 
 licenses(["notice"])  # Apache v2
 
-grpc_package(name = "test/core/util", visibility = "public")
+grpc_package(
+    name = "test/core/util",
+    visibility = "public",
+)
 
 grpc_cc_library(
     name = "gpr_test_util",
@@ -49,6 +52,7 @@ grpc_cc_library(
     name = "grpc_test_util_base",
     srcs = [
         "grpc_profiler.cc",
+        "histogram.cc",
         "mock_endpoint.cc",
         "parse_hexstring.cc",
         "passthru_endpoint.cc",
@@ -62,6 +66,7 @@ grpc_cc_library(
     ],
     hdrs = [
         "grpc_profiler.h",
+        "histogram.h",
         "mock_endpoint.h",
         "parse_hexstring.h",
         "passthru_endpoint.h",
@@ -76,8 +81,8 @@ grpc_cc_library(
     language = "C++",
     deps = [
         ":gpr_test_util",
+        ":grpc_debugger_macros",
         "//:grpc_common",
-        ":grpc_debugger_macros"
     ],
 )
 
@@ -107,13 +112,23 @@ grpc_cc_library(
     name = "fuzzer_corpus_test",
     testonly = 1,
     srcs = ["fuzzer_corpus_test.cc"],
+    external_deps = [
+        "gtest",
+        "gflags",
+    ],
     deps = [
         ":gpr_test_util",
         "//:grpc",
     ],
-    external_deps = [
-        "gtest",
-        "gflags",
+)
+
+grpc_cc_test(
+    name = "histogram_test",
+    srcs = ["histogram_test.cc"],
+    language = "C++",
+    deps = [
+        ":grpc_test_util",
+        "//:gpr",
     ],
 )
 
diff --git a/test/core/util/histogram.cc b/test/core/util/histogram.cc
new file mode 100644
index 0000000000..2f916f831d
--- /dev/null
+++ b/test/core/util/histogram.cc
@@ -0,0 +1,227 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <math.h>
+#include <stddef.h>
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
+#include <grpc/support/useful.h>
+
+#include "test/core/util/histogram.h"
+
+/* Histograms are stored with exponentially increasing bucket sizes.
+   The first bucket is [0, m) where m = 1 + resolution
+   Bucket n (n>=1) contains [m**n, m**(n+1))
+   There are sufficient buckets to reach max_bucket_start */
+
+struct grpc_histogram {
+  /* Sum of all values seen so far */
+  double sum;
+  /* Sum of squares of all values seen so far */
+  double sum_of_squares;
+  /* number of values seen so far */
+  double count;
+  /* m in the description */
+  double multiplier;
+  double one_on_log_multiplier;
+  /* minimum value seen */
+  double min_seen;
+  /* maximum value seen */
+  double max_seen;
+  /* maximum representable value */
+  double max_possible;
+  /* number of buckets */
+  size_t num_buckets;
+  /* the buckets themselves */
+  uint32_t* buckets;
+};
+
+/* determine a bucket index given a value - does no bounds checking */
+static size_t bucket_for_unchecked(grpc_histogram* h, double x) {
+  return (size_t)(log(x) * h->one_on_log_multiplier);
+}
+
+/* bounds checked version of the above */
+static size_t bucket_for(grpc_histogram* h, double x) {
+  size_t bucket = bucket_for_unchecked(h, GPR_CLAMP(x, 1.0, h->max_possible));
+  GPR_ASSERT(bucket < h->num_buckets);
+  return bucket;
+}
+
+/* at what value does a bucket start? */
+static double bucket_start(grpc_histogram* h, double x) {
+  return pow(h->multiplier, x);
+}
+
+grpc_histogram* grpc_histogram_create(double resolution,
+                                      double max_bucket_start) {
+  grpc_histogram* h = (grpc_histogram*)gpr_malloc(sizeof(grpc_histogram));
+  GPR_ASSERT(resolution > 0.0);
+  GPR_ASSERT(max_bucket_start > resolution);
+  h->sum = 0.0;
+  h->sum_of_squares = 0.0;
+  h->multiplier = 1.0 + resolution;
+  h->one_on_log_multiplier = 1.0 / log(1.0 + resolution);
+  h->max_possible = max_bucket_start;
+  h->count = 0.0;
+  h->min_seen = max_bucket_start;
+  h->max_seen = 0.0;
+  h->num_buckets = bucket_for_unchecked(h, max_bucket_start) + 1;
+  GPR_ASSERT(h->num_buckets > 1);
+  GPR_ASSERT(h->num_buckets < 100000000);
+  h->buckets = (uint32_t*)gpr_zalloc(sizeof(uint32_t) * h->num_buckets);
+  return h;
+}
+
+void grpc_histogram_destroy(grpc_histogram* h) {
+  gpr_free(h->buckets);
+  gpr_free(h);
+}
+
+void grpc_histogram_add(grpc_histogram* h, double x) {
+  h->sum += x;
+  h->sum_of_squares += x * x;
+  h->count++;
+  if (x < h->min_seen) {
+    h->min_seen = x;
+  }
+  if (x > h->max_seen) {
+    h->max_seen = x;
+  }
+  h->buckets[bucket_for(h, x)]++;
+}
+
+int grpc_histogram_merge(grpc_histogram* dst, const grpc_histogram* src) {
+  if ((dst->num_buckets != src->num_buckets) ||
+      (dst->multiplier != src->multiplier)) {
+    /* Fail because these histograms don't match */
+    return 0;
+  }
+  grpc_histogram_merge_contents(dst, src->buckets, src->num_buckets,
+                                src->min_seen, src->max_seen, src->sum,
+                                src->sum_of_squares, src->count);
+  return 1;
+}
+
+void grpc_histogram_merge_contents(grpc_histogram* dst, const uint32_t* data,
+                                   size_t data_count, double min_seen,
+                                   double max_seen, double sum,
+                                   double sum_of_squares, double count) {
+  size_t i;
+  GPR_ASSERT(dst->num_buckets == data_count);
+  dst->sum += sum;
+  dst->sum_of_squares += sum_of_squares;
+  dst->count += count;
+  if (min_seen < dst->min_seen) {
+    dst->min_seen = min_seen;
+  }
+  if (max_seen > dst->max_seen) {
+    dst->max_seen = max_seen;
+  }
+  for (i = 0; i < dst->num_buckets; i++) {
+    dst->buckets[i] += data[i];
+  }
+}
+
+static double threshold_for_count_below(grpc_histogram* h, double count_below) {
+  double count_so_far;
+  double lower_bound;
+  double upper_bound;
+  size_t lower_idx;
+  size_t upper_idx;
+
+  if (h->count == 0) {
+    return 0.0;
+  }
+
+  if (count_below <= 0) {
+    return h->min_seen;
+  }
+  if (count_below >= h->count) {
+    return h->max_seen;
+  }
+
+  /* find the lowest bucket that gets us above count_below */
+  count_so_far = 0.0;
+  for (lower_idx = 0; lower_idx < h->num_buckets; lower_idx++) {
+    count_so_far += h->buckets[lower_idx];
+    if (count_so_far >= count_below) {
+      break;
+    }
+  }
+  if (count_so_far == count_below) {
+    /* this bucket hits the threshold exactly... we should be midway through
+       any run of zero values following the bucket */
+    for (upper_idx = lower_idx + 1; upper_idx < h->num_buckets; upper_idx++) {
+      if (h->buckets[upper_idx]) {
+        break;
+      }
+    }
+    return (bucket_start(h, (double)lower_idx) +
+            bucket_start(h, (double)upper_idx)) /
+           2.0;
+  } else {
+    /* treat values as uniform throughout the bucket, and find where this value
+       should lie */
+    lower_bound = bucket_start(h, (double)lower_idx);
+    upper_bound = bucket_start(h, (double)(lower_idx + 1));
+    return GPR_CLAMP(upper_bound - (upper_bound - lower_bound) *
+                                       (count_so_far - count_below) /
+                                       h->buckets[lower_idx],
+                     h->min_seen, h->max_seen);
+  }
+}
+
+double grpc_histogram_percentile(grpc_histogram* h, double percentile) {
+  return threshold_for_count_below(h, h->count * percentile / 100.0);
+}
+
+double grpc_histogram_mean(grpc_histogram* h) {
+  GPR_ASSERT(h->count != 0);
+  return h->sum / h->count;
+}
+
+double grpc_histogram_stddev(grpc_histogram* h) {
+  return sqrt(grpc_histogram_variance(h));
+}
+
+double grpc_histogram_variance(grpc_histogram* h) {
+  if (h->count == 0) return 0.0;
+  return (h->sum_of_squares * h->count - h->sum * h->sum) /
+         (h->count * h->count);
+}
+
+double grpc_histogram_maximum(grpc_histogram* h) { return h->max_seen; }
+
+double grpc_histogram_minimum(grpc_histogram* h) { return h->min_seen; }
+
+double grpc_histogram_count(grpc_histogram* h) { return h->count; }
+
+double grpc_histogram_sum(grpc_histogram* h) { return h->sum; }
+
+double grpc_histogram_sum_of_squares(grpc_histogram* h) {
+  return h->sum_of_squares;
+}
+
+const uint32_t* grpc_histogram_get_contents(grpc_histogram* h, size_t* size) {
+  *size = h->num_buckets;
+  return h->buckets;
+}
diff --git a/test/core/util/histogram.h b/test/core/util/histogram.h
new file mode 100644
index 0000000000..9d4985e64f
--- /dev/null
+++ b/test/core/util/histogram.h
@@ -0,0 +1,62 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_SUPPORT_HISTOGRAM_H
+#define GRPC_SUPPORT_HISTOGRAM_H
+
+#include <grpc/support/port_platform.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct grpc_histogram grpc_histogram;
+
+grpc_histogram* grpc_histogram_create(double resolution,
+                                      double max_bucket_start);
+void grpc_histogram_destroy(grpc_histogram* h);
+void grpc_histogram_add(grpc_histogram* h, double x);
+
+/** The following merges the second histogram into the first. It only works
+   if they have the same buckets and resolution. Returns 0 on failure, 1
+   on success */
+int grpc_histogram_merge(grpc_histogram* dst, const grpc_histogram* src);
+
+double grpc_histogram_percentile(grpc_histogram* histogram, double percentile);
+double grpc_histogram_mean(grpc_histogram* histogram);
+double grpc_histogram_stddev(grpc_histogram* histogram);
+double grpc_histogram_variance(grpc_histogram* histogram);
+double grpc_histogram_maximum(grpc_histogram* histogram);
+double grpc_histogram_minimum(grpc_histogram* histogram);
+double grpc_histogram_count(grpc_histogram* histogram);
+double grpc_histogram_sum(grpc_histogram* histogram);
+double grpc_histogram_sum_of_squares(grpc_histogram* histogram);
+
+const uint32_t* grpc_histogram_get_contents(grpc_histogram* histogram,
+                                            size_t* count);
+void grpc_histogram_merge_contents(grpc_histogram* histogram,
+                                   const uint32_t* data, size_t data_count,
+                                   double min_seen, double max_seen, double sum,
+                                   double sum_of_squares, double count);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* GRPC_SUPPORT_HISTOGRAM_H */
diff --git a/test/core/util/histogram_test.cc b/test/core/util/histogram_test.cc
new file mode 100644
index 0000000000..b96ac7d841
--- /dev/null
+++ b/test/core/util/histogram_test.cc
@@ -0,0 +1,163 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "test/core/util/histogram.h"
+#include <grpc/support/log.h>
+
+#define LOG_TEST(x) gpr_log(GPR_INFO, "%s", x);
+
+static void test_no_op(void) {
+  grpc_histogram_destroy(grpc_histogram_create(0.01, 60e9));
+}
+
+static void expect_percentile(grpc_histogram* h, double percentile,
+                              double min_expect, double max_expect) {
+  double got = grpc_histogram_percentile(h, percentile);
+  gpr_log(GPR_INFO, "@%f%%, expect %f <= %f <= %f", percentile, min_expect, got,
+          max_expect);
+  GPR_ASSERT(min_expect <= got);
+  GPR_ASSERT(got <= max_expect);
+}
+
+static void test_simple(void) {
+  grpc_histogram* h;
+
+  LOG_TEST("test_simple");
+
+  h = grpc_histogram_create(0.01, 60e9);
+  grpc_histogram_add(h, 10000);
+  grpc_histogram_add(h, 10000);
+  grpc_histogram_add(h, 11000);
+  grpc_histogram_add(h, 11000);
+
+  expect_percentile(h, 50, 10001, 10999);
+  GPR_ASSERT(grpc_histogram_mean(h) == 10500);
+
+  grpc_histogram_destroy(h);
+}
+
+static void test_percentile(void) {
+  grpc_histogram* h;
+  double last;
+  double i;
+  double cur;
+
+  LOG_TEST("test_percentile");
+
+  h = grpc_histogram_create(0.05, 1e9);
+  grpc_histogram_add(h, 2.5);
+  grpc_histogram_add(h, 2.5);
+  grpc_histogram_add(h, 8);
+  grpc_histogram_add(h, 4);
+
+  GPR_ASSERT(grpc_histogram_count(h) == 4);
+  GPR_ASSERT(grpc_histogram_minimum(h) == 2.5);
+  GPR_ASSERT(grpc_histogram_maximum(h) == 8);
+  GPR_ASSERT(grpc_histogram_sum(h) == 17);
+  GPR_ASSERT(grpc_histogram_sum_of_squares(h) == 92.5);
+  GPR_ASSERT(grpc_histogram_mean(h) == 4.25);
+  GPR_ASSERT(grpc_histogram_variance(h) == 5.0625);
+  GPR_ASSERT(grpc_histogram_stddev(h) == 2.25);
+
+  expect_percentile(h, -10, 2.5, 2.5);
+  expect_percentile(h, 0, 2.5, 2.5);
+  expect_percentile(h, 12.5, 2.5, 2.5);
+  expect_percentile(h, 25, 2.5, 2.5);
+  expect_percentile(h, 37.5, 2.5, 2.8);
+  expect_percentile(h, 50, 3.0, 3.5);
+  expect_percentile(h, 62.5, 3.5, 4.5);
+  expect_percentile(h, 75, 5, 7.9);
+  expect_percentile(h, 100, 8, 8);
+  expect_percentile(h, 110, 8, 8);
+
+  /* test monotonicity */
+  last = 0.0;
+  for (i = 0; i < 100.0; i += 0.01) {
+    cur = grpc_histogram_percentile(h, i);
+    GPR_ASSERT(cur >= last);
+    last = cur;
+  }
+
+  grpc_histogram_destroy(h);
+}
+
+static void test_merge(void) {
+  grpc_histogram *h1, *h2;
+  double last;
+  double i;
+  double cur;
+
+  LOG_TEST("test_merge");
+
+  h1 = grpc_histogram_create(0.05, 1e9);
+  grpc_histogram_add(h1, 2.5);
+  grpc_histogram_add(h1, 2.5);
+  grpc_histogram_add(h1, 8);
+  grpc_histogram_add(h1, 4);
+
+  h2 = grpc_histogram_create(0.01, 1e9);
+  GPR_ASSERT(grpc_histogram_merge(h1, h2) == 0);
+  grpc_histogram_destroy(h2);
+
+  h2 = grpc_histogram_create(0.05, 1e10);
+  GPR_ASSERT(grpc_histogram_merge(h1, h2) == 0);
+  grpc_histogram_destroy(h2);
+
+  h2 = grpc_histogram_create(0.05, 1e9);
+  GPR_ASSERT(grpc_histogram_merge(h1, h2) == 1);
+  GPR_ASSERT(grpc_histogram_count(h1) == 4);
+  GPR_ASSERT(grpc_histogram_minimum(h1) == 2.5);
+  GPR_ASSERT(grpc_histogram_maximum(h1) == 8);
+  GPR_ASSERT(grpc_histogram_sum(h1) == 17);
+  GPR_ASSERT(grpc_histogram_sum_of_squares(h1) == 92.5);
+  GPR_ASSERT(grpc_histogram_mean(h1) == 4.25);
+  GPR_ASSERT(grpc_histogram_variance(h1) == 5.0625);
+  GPR_ASSERT(grpc_histogram_stddev(h1) == 2.25);
+  grpc_histogram_destroy(h2);
+
+  h2 = grpc_histogram_create(0.05, 1e9);
+  grpc_histogram_add(h2, 7.0);
+  grpc_histogram_add(h2, 17.0);
+  grpc_histogram_add(h2, 1.0);
+  GPR_ASSERT(grpc_histogram_merge(h1, h2) == 1);
+  GPR_ASSERT(grpc_histogram_count(h1) == 7);
+  GPR_ASSERT(grpc_histogram_minimum(h1) == 1.0);
+  GPR_ASSERT(grpc_histogram_maximum(h1) == 17.0);
+  GPR_ASSERT(grpc_histogram_sum(h1) == 42.0);
+  GPR_ASSERT(grpc_histogram_sum_of_squares(h1) == 431.5);
+  GPR_ASSERT(grpc_histogram_mean(h1) == 6.0);
+
+  /* test monotonicity */
+  last = 0.0;
+  for (i = 0; i < 100.0; i += 0.01) {
+    cur = grpc_histogram_percentile(h1, i);
+    GPR_ASSERT(cur >= last);
+    last = cur;
+  }
+
+  grpc_histogram_destroy(h1);
+  grpc_histogram_destroy(h2);
+}
+
+int main(void) {
+  test_no_op();
+  test_simple();
+  test_percentile();
+  test_merge();
+  return 0;
+}