diff options
author | Vijay Pai <vpai@google.com> | 2017-11-30 10:56:06 -0800 |
---|---|---|
committer | Vijay Pai <vpai@google.com> | 2017-11-30 15:02:04 -0800 |
commit | 195cf1ebfd5e3ab12d8271e116e1f022a9b23ef6 (patch) | |
tree | 1fd1ebc2321b09b3de08686f9f73c1bd1b7df2b1 /test/core/util | |
parent | 5f662537deb01539a204273977c7e32394fc3454 (diff) |
Move histogram to test/core/util
Diffstat (limited to 'test/core/util')
-rw-r--r-- | test/core/util/BUILD | 25 | ||||
-rw-r--r-- | test/core/util/histogram.cc | 227 | ||||
-rw-r--r-- | test/core/util/histogram.h | 62 | ||||
-rw-r--r-- | test/core/util/histogram_test.cc | 163 |
4 files changed, 472 insertions, 5 deletions
diff --git a/test/core/util/BUILD b/test/core/util/BUILD index 6443553466..268547f6c9 100644 --- a/test/core/util/BUILD +++ b/test/core/util/BUILD @@ -16,7 +16,10 @@ load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_c licenses(["notice"]) # Apache v2 -grpc_package(name = "test/core/util", visibility = "public") +grpc_package( + name = "test/core/util", + visibility = "public", +) grpc_cc_library( name = "gpr_test_util", @@ -49,6 +52,7 @@ grpc_cc_library( name = "grpc_test_util_base", srcs = [ "grpc_profiler.cc", + "histogram.cc", "mock_endpoint.cc", "parse_hexstring.cc", "passthru_endpoint.cc", @@ -62,6 +66,7 @@ grpc_cc_library( ], hdrs = [ "grpc_profiler.h", + "histogram.h", "mock_endpoint.h", "parse_hexstring.h", "passthru_endpoint.h", @@ -76,8 +81,8 @@ grpc_cc_library( language = "C++", deps = [ ":gpr_test_util", + ":grpc_debugger_macros", "//:grpc_common", - ":grpc_debugger_macros" ], ) @@ -107,13 +112,23 @@ grpc_cc_library( name = "fuzzer_corpus_test", testonly = 1, srcs = ["fuzzer_corpus_test.cc"], + external_deps = [ + "gtest", + "gflags", + ], deps = [ ":gpr_test_util", "//:grpc", ], - external_deps = [ - "gtest", - "gflags", +) + +grpc_cc_test( + name = "histogram_test", + srcs = ["histogram_test.cc"], + language = "C++", + deps = [ + ":grpc_test_util", + "//:gpr", ], ) diff --git a/test/core/util/histogram.cc b/test/core/util/histogram.cc new file mode 100644 index 0000000000..2f916f831d --- /dev/null +++ b/test/core/util/histogram.cc @@ -0,0 +1,227 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <math.h> +#include <stddef.h> +#include <string.h> + +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> +#include <grpc/support/port_platform.h> +#include <grpc/support/useful.h> + +#include "test/core/util/histogram.h" + +/* Histograms are stored with exponentially increasing bucket sizes. + The first bucket is [0, m) where m = 1 + resolution + Bucket n (n>=1) contains [m**n, m**(n+1)) + There are sufficient buckets to reach max_bucket_start */ + +struct grpc_histogram { + /* Sum of all values seen so far */ + double sum; + /* Sum of squares of all values seen so far */ + double sum_of_squares; + /* number of values seen so far */ + double count; + /* m in the description */ + double multiplier; + double one_on_log_multiplier; + /* minimum value seen */ + double min_seen; + /* maximum value seen */ + double max_seen; + /* maximum representable value */ + double max_possible; + /* number of buckets */ + size_t num_buckets; + /* the buckets themselves */ + uint32_t* buckets; +}; + +/* determine a bucket index given a value - does no bounds checking */ +static size_t bucket_for_unchecked(grpc_histogram* h, double x) { + return (size_t)(log(x) * h->one_on_log_multiplier); +} + +/* bounds checked version of the above */ +static size_t bucket_for(grpc_histogram* h, double x) { + size_t bucket = bucket_for_unchecked(h, GPR_CLAMP(x, 1.0, h->max_possible)); + GPR_ASSERT(bucket < h->num_buckets); + return bucket; +} + +/* at what value does a bucket start? */ +static double bucket_start(grpc_histogram* h, double x) { + return pow(h->multiplier, x); +} + +grpc_histogram* grpc_histogram_create(double resolution, + double max_bucket_start) { + grpc_histogram* h = (grpc_histogram*)gpr_malloc(sizeof(grpc_histogram)); + GPR_ASSERT(resolution > 0.0); + GPR_ASSERT(max_bucket_start > resolution); + h->sum = 0.0; + h->sum_of_squares = 0.0; + h->multiplier = 1.0 + resolution; + h->one_on_log_multiplier = 1.0 / log(1.0 + resolution); + h->max_possible = max_bucket_start; + h->count = 0.0; + h->min_seen = max_bucket_start; + h->max_seen = 0.0; + h->num_buckets = bucket_for_unchecked(h, max_bucket_start) + 1; + GPR_ASSERT(h->num_buckets > 1); + GPR_ASSERT(h->num_buckets < 100000000); + h->buckets = (uint32_t*)gpr_zalloc(sizeof(uint32_t) * h->num_buckets); + return h; +} + +void grpc_histogram_destroy(grpc_histogram* h) { + gpr_free(h->buckets); + gpr_free(h); +} + +void grpc_histogram_add(grpc_histogram* h, double x) { + h->sum += x; + h->sum_of_squares += x * x; + h->count++; + if (x < h->min_seen) { + h->min_seen = x; + } + if (x > h->max_seen) { + h->max_seen = x; + } + h->buckets[bucket_for(h, x)]++; +} + +int grpc_histogram_merge(grpc_histogram* dst, const grpc_histogram* src) { + if ((dst->num_buckets != src->num_buckets) || + (dst->multiplier != src->multiplier)) { + /* Fail because these histograms don't match */ + return 0; + } + grpc_histogram_merge_contents(dst, src->buckets, src->num_buckets, + src->min_seen, src->max_seen, src->sum, + src->sum_of_squares, src->count); + return 1; +} + +void grpc_histogram_merge_contents(grpc_histogram* dst, const uint32_t* data, + size_t data_count, double min_seen, + double max_seen, double sum, + double sum_of_squares, double count) { + size_t i; + GPR_ASSERT(dst->num_buckets == data_count); + dst->sum += sum; + dst->sum_of_squares += sum_of_squares; + dst->count += count; + if (min_seen < dst->min_seen) { + dst->min_seen = min_seen; + } + if (max_seen > dst->max_seen) { + dst->max_seen = max_seen; + } + for (i = 0; i < dst->num_buckets; i++) { + dst->buckets[i] += data[i]; + } +} + +static double threshold_for_count_below(grpc_histogram* h, double count_below) { + double count_so_far; + double lower_bound; + double upper_bound; + size_t lower_idx; + size_t upper_idx; + + if (h->count == 0) { + return 0.0; + } + + if (count_below <= 0) { + return h->min_seen; + } + if (count_below >= h->count) { + return h->max_seen; + } + + /* find the lowest bucket that gets us above count_below */ + count_so_far = 0.0; + for (lower_idx = 0; lower_idx < h->num_buckets; lower_idx++) { + count_so_far += h->buckets[lower_idx]; + if (count_so_far >= count_below) { + break; + } + } + if (count_so_far == count_below) { + /* this bucket hits the threshold exactly... we should be midway through + any run of zero values following the bucket */ + for (upper_idx = lower_idx + 1; upper_idx < h->num_buckets; upper_idx++) { + if (h->buckets[upper_idx]) { + break; + } + } + return (bucket_start(h, (double)lower_idx) + + bucket_start(h, (double)upper_idx)) / + 2.0; + } else { + /* treat values as uniform throughout the bucket, and find where this value + should lie */ + lower_bound = bucket_start(h, (double)lower_idx); + upper_bound = bucket_start(h, (double)(lower_idx + 1)); + return GPR_CLAMP(upper_bound - (upper_bound - lower_bound) * + (count_so_far - count_below) / + h->buckets[lower_idx], + h->min_seen, h->max_seen); + } +} + +double grpc_histogram_percentile(grpc_histogram* h, double percentile) { + return threshold_for_count_below(h, h->count * percentile / 100.0); +} + +double grpc_histogram_mean(grpc_histogram* h) { + GPR_ASSERT(h->count != 0); + return h->sum / h->count; +} + +double grpc_histogram_stddev(grpc_histogram* h) { + return sqrt(grpc_histogram_variance(h)); +} + +double grpc_histogram_variance(grpc_histogram* h) { + if (h->count == 0) return 0.0; + return (h->sum_of_squares * h->count - h->sum * h->sum) / + (h->count * h->count); +} + +double grpc_histogram_maximum(grpc_histogram* h) { return h->max_seen; } + +double grpc_histogram_minimum(grpc_histogram* h) { return h->min_seen; } + +double grpc_histogram_count(grpc_histogram* h) { return h->count; } + +double grpc_histogram_sum(grpc_histogram* h) { return h->sum; } + +double grpc_histogram_sum_of_squares(grpc_histogram* h) { + return h->sum_of_squares; +} + +const uint32_t* grpc_histogram_get_contents(grpc_histogram* h, size_t* size) { + *size = h->num_buckets; + return h->buckets; +} diff --git a/test/core/util/histogram.h b/test/core/util/histogram.h new file mode 100644 index 0000000000..9d4985e64f --- /dev/null +++ b/test/core/util/histogram.h @@ -0,0 +1,62 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_SUPPORT_HISTOGRAM_H +#define GRPC_SUPPORT_HISTOGRAM_H + +#include <grpc/support/port_platform.h> +#include <stddef.h> + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct grpc_histogram grpc_histogram; + +grpc_histogram* grpc_histogram_create(double resolution, + double max_bucket_start); +void grpc_histogram_destroy(grpc_histogram* h); +void grpc_histogram_add(grpc_histogram* h, double x); + +/** The following merges the second histogram into the first. It only works + if they have the same buckets and resolution. Returns 0 on failure, 1 + on success */ +int grpc_histogram_merge(grpc_histogram* dst, const grpc_histogram* src); + +double grpc_histogram_percentile(grpc_histogram* histogram, double percentile); +double grpc_histogram_mean(grpc_histogram* histogram); +double grpc_histogram_stddev(grpc_histogram* histogram); +double grpc_histogram_variance(grpc_histogram* histogram); +double grpc_histogram_maximum(grpc_histogram* histogram); +double grpc_histogram_minimum(grpc_histogram* histogram); +double grpc_histogram_count(grpc_histogram* histogram); +double grpc_histogram_sum(grpc_histogram* histogram); +double grpc_histogram_sum_of_squares(grpc_histogram* histogram); + +const uint32_t* grpc_histogram_get_contents(grpc_histogram* histogram, + size_t* count); +void grpc_histogram_merge_contents(grpc_histogram* histogram, + const uint32_t* data, size_t data_count, + double min_seen, double max_seen, double sum, + double sum_of_squares, double count); + +#ifdef __cplusplus +} +#endif + +#endif /* GRPC_SUPPORT_HISTOGRAM_H */ diff --git a/test/core/util/histogram_test.cc b/test/core/util/histogram_test.cc new file mode 100644 index 0000000000..b96ac7d841 --- /dev/null +++ b/test/core/util/histogram_test.cc @@ -0,0 +1,163 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "test/core/util/histogram.h" +#include <grpc/support/log.h> + +#define LOG_TEST(x) gpr_log(GPR_INFO, "%s", x); + +static void test_no_op(void) { + grpc_histogram_destroy(grpc_histogram_create(0.01, 60e9)); +} + +static void expect_percentile(grpc_histogram* h, double percentile, + double min_expect, double max_expect) { + double got = grpc_histogram_percentile(h, percentile); + gpr_log(GPR_INFO, "@%f%%, expect %f <= %f <= %f", percentile, min_expect, got, + max_expect); + GPR_ASSERT(min_expect <= got); + GPR_ASSERT(got <= max_expect); +} + +static void test_simple(void) { + grpc_histogram* h; + + LOG_TEST("test_simple"); + + h = grpc_histogram_create(0.01, 60e9); + grpc_histogram_add(h, 10000); + grpc_histogram_add(h, 10000); + grpc_histogram_add(h, 11000); + grpc_histogram_add(h, 11000); + + expect_percentile(h, 50, 10001, 10999); + GPR_ASSERT(grpc_histogram_mean(h) == 10500); + + grpc_histogram_destroy(h); +} + +static void test_percentile(void) { + grpc_histogram* h; + double last; + double i; + double cur; + + LOG_TEST("test_percentile"); + + h = grpc_histogram_create(0.05, 1e9); + grpc_histogram_add(h, 2.5); + grpc_histogram_add(h, 2.5); + grpc_histogram_add(h, 8); + grpc_histogram_add(h, 4); + + GPR_ASSERT(grpc_histogram_count(h) == 4); + GPR_ASSERT(grpc_histogram_minimum(h) == 2.5); + GPR_ASSERT(grpc_histogram_maximum(h) == 8); + GPR_ASSERT(grpc_histogram_sum(h) == 17); + GPR_ASSERT(grpc_histogram_sum_of_squares(h) == 92.5); + GPR_ASSERT(grpc_histogram_mean(h) == 4.25); + GPR_ASSERT(grpc_histogram_variance(h) == 5.0625); + GPR_ASSERT(grpc_histogram_stddev(h) == 2.25); + + expect_percentile(h, -10, 2.5, 2.5); + expect_percentile(h, 0, 2.5, 2.5); + expect_percentile(h, 12.5, 2.5, 2.5); + expect_percentile(h, 25, 2.5, 2.5); + expect_percentile(h, 37.5, 2.5, 2.8); + expect_percentile(h, 50, 3.0, 3.5); + expect_percentile(h, 62.5, 3.5, 4.5); + expect_percentile(h, 75, 5, 7.9); + expect_percentile(h, 100, 8, 8); + expect_percentile(h, 110, 8, 8); + + /* test monotonicity */ + last = 0.0; + for (i = 0; i < 100.0; i += 0.01) { + cur = grpc_histogram_percentile(h, i); + GPR_ASSERT(cur >= last); + last = cur; + } + + grpc_histogram_destroy(h); +} + +static void test_merge(void) { + grpc_histogram *h1, *h2; + double last; + double i; + double cur; + + LOG_TEST("test_merge"); + + h1 = grpc_histogram_create(0.05, 1e9); + grpc_histogram_add(h1, 2.5); + grpc_histogram_add(h1, 2.5); + grpc_histogram_add(h1, 8); + grpc_histogram_add(h1, 4); + + h2 = grpc_histogram_create(0.01, 1e9); + GPR_ASSERT(grpc_histogram_merge(h1, h2) == 0); + grpc_histogram_destroy(h2); + + h2 = grpc_histogram_create(0.05, 1e10); + GPR_ASSERT(grpc_histogram_merge(h1, h2) == 0); + grpc_histogram_destroy(h2); + + h2 = grpc_histogram_create(0.05, 1e9); + GPR_ASSERT(grpc_histogram_merge(h1, h2) == 1); + GPR_ASSERT(grpc_histogram_count(h1) == 4); + GPR_ASSERT(grpc_histogram_minimum(h1) == 2.5); + GPR_ASSERT(grpc_histogram_maximum(h1) == 8); + GPR_ASSERT(grpc_histogram_sum(h1) == 17); + GPR_ASSERT(grpc_histogram_sum_of_squares(h1) == 92.5); + GPR_ASSERT(grpc_histogram_mean(h1) == 4.25); + GPR_ASSERT(grpc_histogram_variance(h1) == 5.0625); + GPR_ASSERT(grpc_histogram_stddev(h1) == 2.25); + grpc_histogram_destroy(h2); + + h2 = grpc_histogram_create(0.05, 1e9); + grpc_histogram_add(h2, 7.0); + grpc_histogram_add(h2, 17.0); + grpc_histogram_add(h2, 1.0); + GPR_ASSERT(grpc_histogram_merge(h1, h2) == 1); + GPR_ASSERT(grpc_histogram_count(h1) == 7); + GPR_ASSERT(grpc_histogram_minimum(h1) == 1.0); + GPR_ASSERT(grpc_histogram_maximum(h1) == 17.0); + GPR_ASSERT(grpc_histogram_sum(h1) == 42.0); + GPR_ASSERT(grpc_histogram_sum_of_squares(h1) == 431.5); + GPR_ASSERT(grpc_histogram_mean(h1) == 6.0); + + /* test monotonicity */ + last = 0.0; + for (i = 0; i < 100.0; i += 0.01) { + cur = grpc_histogram_percentile(h1, i); + GPR_ASSERT(cur >= last); + last = cur; + } + + grpc_histogram_destroy(h1); + grpc_histogram_destroy(h2); +} + +int main(void) { + test_no_op(); + test_simple(); + test_percentile(); + test_merge(); + return 0; +} |