Merge remote-tracking branch 'upstream/master' into cares_bazel_rule

17 files changed, 2171 insertions, 1183 deletions

diff --git a/test/cpp/microbenchmarks/bm_arena.cc b/test/cpp/microbenchmarks/bm_arena.cc
new file mode 100644
index 0000000000..770c0b6d47
--- /dev/null
+++ b/test/cpp/microbenchmarks/bm_arena.cc
@@ -0,0 +1,76 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* Benchmark arenas */
+
+extern "C" {
+#include "src/core/lib/support/arena.h"
+}
+#include "test/cpp/microbenchmarks/helpers.h"
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+
+static void BM_Arena_NoOp(benchmark::State& state) {
+  while (state.KeepRunning()) {
+    gpr_arena_destroy(gpr_arena_create(state.range(0)));
+  }
+}
+BENCHMARK(BM_Arena_NoOp)->Range(1, 1024 * 1024);
+
+static void BM_Arena_ManyAlloc(benchmark::State& state) {
+  gpr_arena* a = gpr_arena_create(state.range(0));
+  const size_t realloc_after =
+      1024 * 1024 * 1024 / ((state.range(1) + 15) & 0xffffff0u);
+  while (state.KeepRunning()) {
+    gpr_arena_alloc(a, state.range(1));
+    // periodically recreate arena to avoid OOM
+    if (state.iterations() % realloc_after == 0) {
+      gpr_arena_destroy(a);
+      a = gpr_arena_create(state.range(0));
+    }
+  }
+  gpr_arena_destroy(a);
+}
+BENCHMARK(BM_Arena_ManyAlloc)->Ranges({{1, 1024 * 1024}, {1, 32 * 1024}});
+
+static void BM_Arena_Batch(benchmark::State& state) {
+  while (state.KeepRunning()) {
+    gpr_arena* a = gpr_arena_create(state.range(0));
+    for (int i = 0; i < state.range(1); i++) {
+      gpr_arena_alloc(a, state.range(2));
+    }
+    gpr_arena_destroy(a);
+  }
+}
+BENCHMARK(BM_Arena_Batch)->Ranges({{1, 64 * 1024}, {1, 64}, {1, 1024}});
+
+BENCHMARK_MAIN();
diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc
index a633b49c65..5ef40abb97 100644
--- a/test/cpp/microbenchmarks/bm_call_create.cc
+++ b/test/cpp/microbenchmarks/bm_call_create.cc
@@ -37,6 +37,7 @@
 #include <string.h>
 #include <sstream>
 
+#include <grpc++/channel.h>
 #include <grpc++/support/channel_arguments.h>
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
@@ -55,28 +56,106 @@ extern "C" {
 #include "src/core/lib/transport/transport_impl.h"
 }
 
+#include "src/cpp/client/create_channel_internal.h"
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
+#include "test/cpp/microbenchmarks/helpers.h"
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
-static struct Init {
-  Init() { grpc_init(); }
-  ~Init() { grpc_shutdown(); }
-} g_init;
+auto &force_library_initialization = Library::get();
 
-static void BM_InsecureChannelWithDefaults(benchmark::State &state) {
-  grpc_channel *channel =
-      grpc_insecure_channel_create("localhost:12345", NULL, NULL);
+void BM_Zalloc(benchmark::State &state) {
+  // speed of light for call creation is zalloc, so benchmark a few interesting
+  // sizes
+  size_t sz = state.range(0);
+  while (state.KeepRunning()) {
+    gpr_free(gpr_zalloc(sz));
+  }
+}
+BENCHMARK(BM_Zalloc)
+    ->Arg(64)
+    ->Arg(128)
+    ->Arg(256)
+    ->Arg(512)
+    ->Arg(1024)
+    ->Arg(1536)
+    ->Arg(2048)
+    ->Arg(3072)
+    ->Arg(4096)
+    ->Arg(5120)
+    ->Arg(6144)
+    ->Arg(7168);
+
+class BaseChannelFixture {
+ public:
+  BaseChannelFixture(grpc_channel *channel) : channel_(channel) {}
+  ~BaseChannelFixture() { grpc_channel_destroy(channel_); }
+
+  grpc_channel *channel() const { return channel_; }
+
+ private:
+  grpc_channel *const channel_;
+};
+
+class InsecureChannel : public BaseChannelFixture {
+ public:
+  InsecureChannel()
+      : BaseChannelFixture(
+            grpc_insecure_channel_create("localhost:1234", NULL, NULL)) {}
+};
+
+class LameChannel : public BaseChannelFixture {
+ public:
+  LameChannel()
+      : BaseChannelFixture(grpc_lame_client_channel_create(
+            "localhost:1234", GRPC_STATUS_UNAUTHENTICATED, "blah")) {}
+};
+
+template <class Fixture>
+static void BM_CallCreateDestroy(benchmark::State &state) {
+  TrackCounters track_counters;
+  Fixture fixture;
   grpc_completion_queue *cq = grpc_completion_queue_create(NULL);
-  grpc_slice method = grpc_slice_from_static_string("/foo/bar");
   gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
+  void *method_hdl =
+      grpc_channel_register_call(fixture.channel(), "/foo/bar", NULL, NULL);
   while (state.KeepRunning()) {
-    grpc_call_destroy(grpc_channel_create_call(channel, NULL,
-                                               GRPC_PROPAGATE_DEFAULTS, cq,
-                                               method, NULL, deadline, NULL));
+    grpc_call_destroy(grpc_channel_create_registered_call(
+        fixture.channel(), NULL, GRPC_PROPAGATE_DEFAULTS, cq, method_hdl,
+        deadline, NULL));
   }
-  grpc_channel_destroy(channel);
   grpc_completion_queue_destroy(cq);
+  track_counters.Finish(state);
+}
+
+BENCHMARK_TEMPLATE(BM_CallCreateDestroy, InsecureChannel);
+BENCHMARK_TEMPLATE(BM_CallCreateDestroy, LameChannel);
+
+static void *tag(int i) {
+  return reinterpret_cast<void *>(static_cast<intptr_t>(i));
+}
+
+static void BM_LameChannelCallCreateCpp(benchmark::State &state) {
+  TrackCounters track_counters;
+  auto stub =
+      grpc::testing::EchoTestService::NewStub(grpc::CreateChannelInternal(
+          "", grpc_lame_client_channel_create(
+                  "localhost:1234", GRPC_STATUS_UNAUTHENTICATED, "blah")));
+  grpc::CompletionQueue cq;
+  grpc::testing::EchoRequest send_request;
+  grpc::testing::EchoResponse recv_response;
+  grpc::Status recv_status;
+  while (state.KeepRunning()) {
+    grpc::ClientContext cli_ctx;
+    auto reader = stub->AsyncEcho(&cli_ctx, send_request, &cq);
+    reader->Finish(&recv_response, &recv_status, tag(0));
+    void *t;
+    bool ok;
+    GPR_ASSERT(cq.Next(&t, &ok));
+    GPR_ASSERT(ok);
+  }
+  track_counters.Finish(state);
 }
-BENCHMARK(BM_InsecureChannelWithDefaults);
+BENCHMARK(BM_LameChannelCallCreateCpp);
 
 static void FilterDestroy(grpc_exec_ctx *exec_ctx, void *arg,
                           grpc_error *error) {
@@ -153,7 +232,7 @@ static void SetPollsetOrPollsetSet(grpc_exec_ctx *exec_ctx,
 
 static void DestroyCallElem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                             const grpc_call_final_info *final_info,
-                            void *and_free_memory) {}
+                            grpc_closure *then_sched_closure) {}
 
 grpc_error *InitChannelElem(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem,
                             grpc_channel_element_args *args) {
@@ -196,7 +275,7 @@ const char *name;
 /* implementation of grpc_transport_init_stream */
 int InitStream(grpc_exec_ctx *exec_ctx, grpc_transport *self,
                grpc_stream *stream, grpc_stream_refcount *refcount,
-               const void *server_data) {
+               const void *server_data, gpr_arena *arena) {
   return 0;
 }
 
@@ -220,7 +299,7 @@ void PerformOp(grpc_exec_ctx *exec_ctx, grpc_transport *self,
 
 /* implementation of grpc_transport_destroy_stream */
 void DestroyStream(grpc_exec_ctx *exec_ctx, grpc_transport *self,
-                   grpc_stream *stream, void *and_free_memory) {}
+                   grpc_stream *stream, grpc_closure *then_sched_closure) {}
 
 /* implementation of grpc_transport_destroy */
 void Destroy(grpc_exec_ctx *exec_ctx, grpc_transport *self) {}
@@ -289,6 +368,7 @@ class SendEmptyMetadata {
 // perform on said filter.
 template <class Fixture, class TestOp>
 static void BM_IsolatedFilter(benchmark::State &state) {
+  TrackCounters track_counters;
   Fixture fixture;
   std::ostringstream label;
 
@@ -312,9 +392,9 @@ static void BM_IsolatedFilter(benchmark::State &state) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   size_t channel_size = grpc_channel_stack_size(&filters[0], filters.size());
   grpc_channel_stack *channel_stack =
-      static_cast<grpc_channel_stack *>(gpr_malloc(channel_size));
+      static_cast<grpc_channel_stack *>(gpr_zalloc(channel_size));
   GPR_ASSERT(GRPC_LOG_IF_ERROR(
-      "call_stack_init",
+      "channel_stack_init",
       grpc_channel_stack_init(&exec_ctx, 1, FilterDestroy, channel_stack,
                               &filters[0], filters.size(), &channel_args,
                               fixture.flags & REQUIRES_TRANSPORT
@@ -323,27 +403,42 @@ static void BM_IsolatedFilter(benchmark::State &state) {
                               "CHANNEL", channel_stack)));
   grpc_exec_ctx_flush(&exec_ctx);
   grpc_call_stack *call_stack = static_cast<grpc_call_stack *>(
-      gpr_malloc(channel_stack->call_stack_size));
+      gpr_zalloc(channel_stack->call_stack_size));
   gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
   gpr_timespec start_time = gpr_now(GPR_CLOCK_MONOTONIC);
   grpc_slice method = grpc_slice_from_static_string("/foo/bar");
   grpc_call_final_info final_info;
   TestOp test_op_data;
+  grpc_call_element_args call_args;
+  call_args.call_stack = call_stack;
+  call_args.server_transport_data = NULL;
+  call_args.context = NULL;
+  call_args.path = method;
+  call_args.start_time = start_time;
+  call_args.deadline = deadline;
+  const int kArenaSize = 4096;
+  call_args.arena = gpr_arena_create(kArenaSize);
   while (state.KeepRunning()) {
     GRPC_ERROR_UNREF(grpc_call_stack_init(&exec_ctx, channel_stack, 1,
-                                          DoNothing, NULL, NULL, NULL, method,
-                                          start_time, deadline, call_stack));
+                                          DoNothing, NULL, &call_args));
     typename TestOp::Op op(&exec_ctx, &test_op_data, call_stack);
     grpc_call_stack_destroy(&exec_ctx, call_stack, &final_info, NULL);
     op.Finish(&exec_ctx);
     grpc_exec_ctx_flush(&exec_ctx);
+    // recreate arena every 64k iterations to avoid oom
+    if (0 == (state.iterations() & 0xffff)) {
+      gpr_arena_destroy(call_args.arena);
+      call_args.arena = gpr_arena_create(kArenaSize);
+    }
   }
+  gpr_arena_destroy(call_args.arena);
   grpc_channel_stack_destroy(&exec_ctx, channel_stack);
   grpc_exec_ctx_finish(&exec_ctx);
   gpr_free(channel_stack);
   gpr_free(call_stack);
 
   state.SetLabel(label.str());
+  track_counters.Finish(state);
 }
 
 typedef Fixture<nullptr, 0> NoFilter;
diff --git a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc
index 5fb3f37130..563db758f7 100644
--- a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc
+++ b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc
@@ -42,18 +42,17 @@ extern "C" {
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/transport/static_metadata.h"
 }
+#include "test/cpp/microbenchmarks/helpers.h"
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
-static struct Init {
-  Init() { grpc_init(); }
-  ~Init() { grpc_shutdown(); }
-} g_init;
+auto &force_library_initialization = Library::get();
 
 ////////////////////////////////////////////////////////////////////////////////
 // HPACK encoder
 //
 
 static void BM_HpackEncoderInitDestroy(benchmark::State &state) {
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_chttp2_hpack_compressor c;
   while (state.KeepRunning()) {
@@ -62,11 +61,13 @@ static void BM_HpackEncoderInitDestroy(benchmark::State &state) {
     grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_HpackEncoderInitDestroy);
 
 template <class Fixture>
 static void BM_HpackEncoderEncodeHeader(benchmark::State &state) {
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
 
   grpc_metadata_batch b;
@@ -102,6 +103,7 @@ static void BM_HpackEncoderEncodeHeader(benchmark::State &state) {
         << " header_bytes/iter:" << (static_cast<double>(stats.header_bytes) /
                                      static_cast<double>(state.iterations()));
   state.SetLabel(label.str());
+  track_counters.Finish(state);
 }
 
 namespace hpack_encoder_fixtures {
@@ -216,6 +218,7 @@ BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader,
 //
 
 static void BM_HpackParserInitDestroy(benchmark::State &state) {
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_chttp2_hpack_parser p;
   while (state.KeepRunning()) {
@@ -224,6 +227,7 @@ static void BM_HpackParserInitDestroy(benchmark::State &state) {
     grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_HpackParserInitDestroy);
 
@@ -234,6 +238,7 @@ static void UnrefHeader(grpc_exec_ctx *exec_ctx, void *user_data,
 
 template <class Fixture>
 static void BM_HpackParserParseHeader(benchmark::State &state) {
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   std::vector<grpc_slice> init_slices = Fixture::GetInitSlices();
   std::vector<grpc_slice> benchmark_slices = Fixture::GetBenchmarkSlices();
@@ -252,6 +257,7 @@ static void BM_HpackParserParseHeader(benchmark::State &state) {
   }
   grpc_chttp2_hpack_parser_destroy(&exec_ctx, &p);
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 
 namespace hpack_parser_fixtures {
diff --git a/test/cpp/microbenchmarks/bm_closure.cc b/test/cpp/microbenchmarks/bm_closure.cc
index 03aede35b2..28a385b6c1 100644
--- a/test/cpp/microbenchmarks/bm_closure.cc
+++ b/test/cpp/microbenchmarks/bm_closure.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2017, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -39,88 +39,50 @@ extern "C" {
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/support/spinlock.h"
 }
 
+#include "test/cpp/microbenchmarks/helpers.h"
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
-#include <sstream>
-
-#ifdef GPR_LOW_LEVEL_COUNTERS
-extern "C" gpr_atm gpr_mu_locks;
-#endif
-
-static class InitializeStuff {
- public:
-  InitializeStuff() { grpc_init(); }
-  ~InitializeStuff() { grpc_shutdown(); }
-} initialize_stuff;
-
-class TrackCounters {
- public:
-  TrackCounters(benchmark::State& state) : state_(state) {}
-
-  ~TrackCounters() {
-    std::ostringstream out;
-#ifdef GPR_LOW_LEVEL_COUNTERS
-    out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) -
-                                       mu_locks_at_start_) /
-                              (double)state_.iterations())
-        << " atm_cas/iter:"
-        << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) -
-                     atm_cas_at_start_) /
-            (double)state_.iterations())
-        << " atm_add/iter:"
-        << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) -
-                     atm_add_at_start_) /
-            (double)state_.iterations());
-#endif
-    state_.SetLabel(out.str());
-  }
-
- private:
-  benchmark::State& state_;
-#ifdef GPR_LOW_LEVEL_COUNTERS
-  const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks);
-  const size_t atm_cas_at_start_ =
-      gpr_atm_no_barrier_load(&gpr_counter_atm_cas);
-  const size_t atm_add_at_start_ =
-      gpr_atm_no_barrier_load(&gpr_counter_atm_add);
-#endif
-};
+auto& force_library_initialization = Library::get();
 
 static void BM_NoOpExecCtx(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
     grpc_exec_ctx_finish(&exec_ctx);
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_NoOpExecCtx);
 
 static void BM_WellFlushed(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   while (state.KeepRunning()) {
     grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_WellFlushed);
 
 static void DoNothing(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {}
 
 static void BM_ClosureInitAgainstExecCtx(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_closure c;
   while (state.KeepRunning()) {
     benchmark::DoNotOptimize(
         grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx));
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureInitAgainstExecCtx);
 
 static void BM_ClosureInitAgainstCombiner(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   grpc_closure c;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -130,11 +92,12 @@ static void BM_ClosureInitAgainstCombiner(benchmark::State& state) {
   }
   GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureInitAgainstCombiner);
 
 static void BM_ClosureRunOnExecCtx(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_closure c;
   grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -143,11 +106,12 @@ static void BM_ClosureRunOnExecCtx(benchmark::State& state) {
     grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureRunOnExecCtx);
 
 static void BM_ClosureCreateAndRun(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   while (state.KeepRunning()) {
     grpc_closure_run(&exec_ctx, grpc_closure_create(DoNothing, NULL,
@@ -155,11 +119,12 @@ static void BM_ClosureCreateAndRun(benchmark::State& state) {
                      GRPC_ERROR_NONE);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureCreateAndRun);
 
 static void BM_ClosureInitAndRun(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_closure c;
   while (state.KeepRunning()) {
@@ -168,11 +133,12 @@ static void BM_ClosureInitAndRun(benchmark::State& state) {
                      GRPC_ERROR_NONE);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureInitAndRun);
 
 static void BM_ClosureSchedOnExecCtx(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_closure c;
   grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -181,11 +147,12 @@ static void BM_ClosureSchedOnExecCtx(benchmark::State& state) {
     grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSchedOnExecCtx);
 
 static void BM_ClosureSched2OnExecCtx(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_closure c1;
   grpc_closure c2;
   grpc_closure_init(&c1, DoNothing, NULL, grpc_schedule_on_exec_ctx);
@@ -197,11 +164,12 @@ static void BM_ClosureSched2OnExecCtx(benchmark::State& state) {
     grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSched2OnExecCtx);
 
 static void BM_ClosureSched3OnExecCtx(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_closure c1;
   grpc_closure c2;
   grpc_closure c3;
@@ -216,11 +184,12 @@ static void BM_ClosureSched3OnExecCtx(benchmark::State& state) {
     grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSched3OnExecCtx);
 
 static void BM_AcquireMutex(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   // for comparison with the combiner stuff below
   gpr_mu mu;
   gpr_mu_init(&mu);
@@ -231,11 +200,64 @@ static void BM_AcquireMutex(benchmark::State& state) {
     gpr_mu_unlock(&mu);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_AcquireMutex);
 
+static void BM_TryAcquireMutex(benchmark::State& state) {
+  TrackCounters track_counters;
+  // for comparison with the combiner stuff below
+  gpr_mu mu;
+  gpr_mu_init(&mu);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  while (state.KeepRunning()) {
+    if (gpr_mu_trylock(&mu)) {
+      DoNothing(&exec_ctx, NULL, GRPC_ERROR_NONE);
+      gpr_mu_unlock(&mu);
+    } else {
+      abort();
+    }
+  }
+  grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_TryAcquireMutex);
+
+static void BM_AcquireSpinlock(benchmark::State& state) {
+  TrackCounters track_counters;
+  // for comparison with the combiner stuff below
+  gpr_spinlock mu = GPR_SPINLOCK_INITIALIZER;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  while (state.KeepRunning()) {
+    gpr_spinlock_lock(&mu);
+    DoNothing(&exec_ctx, NULL, GRPC_ERROR_NONE);
+    gpr_spinlock_unlock(&mu);
+  }
+  grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_AcquireSpinlock);
+
+static void BM_TryAcquireSpinlock(benchmark::State& state) {
+  TrackCounters track_counters;
+  // for comparison with the combiner stuff below
+  gpr_spinlock mu = GPR_SPINLOCK_INITIALIZER;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  while (state.KeepRunning()) {
+    if (gpr_spinlock_trylock(&mu)) {
+      DoNothing(&exec_ctx, NULL, GRPC_ERROR_NONE);
+      gpr_spinlock_unlock(&mu);
+    } else {
+      abort();
+    }
+  }
+  grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_TryAcquireSpinlock);
+
 static void BM_ClosureSchedOnCombiner(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   grpc_closure c;
   grpc_closure_init(&c, DoNothing, NULL,
@@ -247,11 +269,12 @@ static void BM_ClosureSchedOnCombiner(benchmark::State& state) {
   }
   GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSchedOnCombiner);
 
 static void BM_ClosureSched2OnCombiner(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   grpc_closure c1;
   grpc_closure c2;
@@ -267,11 +290,12 @@ static void BM_ClosureSched2OnCombiner(benchmark::State& state) {
   }
   GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSched2OnCombiner);
 
 static void BM_ClosureSched3OnCombiner(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   grpc_closure c1;
   grpc_closure c2;
@@ -291,11 +315,12 @@ static void BM_ClosureSched3OnCombiner(benchmark::State& state) {
   }
   GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSched3OnCombiner);
 
 static void BM_ClosureSched2OnTwoCombiners(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_combiner* combiner1 = grpc_combiner_create(NULL);
   grpc_combiner* combiner2 = grpc_combiner_create(NULL);
   grpc_closure c1;
@@ -313,11 +338,12 @@ static void BM_ClosureSched2OnTwoCombiners(benchmark::State& state) {
   GRPC_COMBINER_UNREF(&exec_ctx, combiner1, "finished");
   GRPC_COMBINER_UNREF(&exec_ctx, combiner2, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSched2OnTwoCombiners);
 
 static void BM_ClosureSched4OnTwoCombiners(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_combiner* combiner1 = grpc_combiner_create(NULL);
   grpc_combiner* combiner2 = grpc_combiner_create(NULL);
   grpc_closure c1;
@@ -343,6 +369,7 @@ static void BM_ClosureSched4OnTwoCombiners(benchmark::State& state) {
   GRPC_COMBINER_UNREF(&exec_ctx, combiner1, "finished");
   GRPC_COMBINER_UNREF(&exec_ctx, combiner2, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSched4OnTwoCombiners);
 
@@ -378,16 +405,17 @@ class Rescheduler {
 };
 
 static void BM_ClosureReschedOnExecCtx(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   Rescheduler r(state, grpc_schedule_on_exec_ctx);
   r.ScheduleFirst(&exec_ctx);
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureReschedOnExecCtx);
 
 static void BM_ClosureReschedOnCombiner(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   Rescheduler r(state, grpc_combiner_scheduler(combiner, false));
@@ -395,11 +423,12 @@ static void BM_ClosureReschedOnCombiner(benchmark::State& state) {
   grpc_exec_ctx_flush(&exec_ctx);
   GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureReschedOnCombiner);
 
 static void BM_ClosureReschedOnCombinerFinally(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   Rescheduler r(state, grpc_combiner_finally_scheduler(combiner, false));
@@ -408,6 +437,7 @@ static void BM_ClosureReschedOnCombinerFinally(benchmark::State& state) {
   grpc_exec_ctx_flush(&exec_ctx);
   GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureReschedOnCombinerFinally);
 
diff --git a/test/cpp/microbenchmarks/bm_cq.cc b/test/cpp/microbenchmarks/bm_cq.cc
index 195dcef3ab..91e6a85101 100644
--- a/test/cpp/microbenchmarks/bm_cq.cc
+++ b/test/cpp/microbenchmarks/bm_cq.cc
@@ -38,6 +38,7 @@
 #include <grpc++/impl/grpc_library.h>
 #include <grpc/grpc.h>
 
+#include "test/cpp/microbenchmarks/helpers.h"
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
 extern "C" {
@@ -47,27 +48,23 @@ extern "C" {
 namespace grpc {
 namespace testing {
 
-static class InitializeStuff {
- public:
-  InitializeStuff() { init_lib_.init(); }
-  ~InitializeStuff() { init_lib_.shutdown(); }
-
- private:
-  internal::GrpcLibrary init_lib_;
-  internal::GrpcLibraryInitializer init_;
-} initialize_stuff;
+auto& force_library_initialization = Library::get();
 
 static void BM_CreateDestroyCpp(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     CompletionQueue cq;
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_CreateDestroyCpp);
 
 static void BM_CreateDestroyCore(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     grpc_completion_queue_destroy(grpc_completion_queue_create(NULL));
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_CreateDestroyCore);
 
@@ -80,6 +77,7 @@ class DummyTag final : public CompletionQueueTag {
 };
 
 static void BM_Pass1Cpp(benchmark::State& state) {
+  TrackCounters track_counters;
   CompletionQueue cq;
   grpc_completion_queue* c_cq = cq.cq();
   while (state.KeepRunning()) {
@@ -94,10 +92,12 @@ static void BM_Pass1Cpp(benchmark::State& state) {
     bool ok;
     cq.Next(&tag, &ok);
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_Pass1Cpp);
 
 static void BM_Pass1Core(benchmark::State& state) {
+  TrackCounters track_counters;
   grpc_completion_queue* cq = grpc_completion_queue_create(NULL);
   gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
   while (state.KeepRunning()) {
@@ -110,9 +110,40 @@ static void BM_Pass1Core(benchmark::State& state) {
     grpc_completion_queue_next(cq, deadline, NULL);
   }
   grpc_completion_queue_destroy(cq);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_Pass1Core);
 
+static void BM_Pluck1Core(benchmark::State& state) {
+  TrackCounters track_counters;
+  grpc_completion_queue* cq = grpc_completion_queue_create(NULL);
+  gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
+  while (state.KeepRunning()) {
+    grpc_cq_completion completion;
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    grpc_cq_begin_op(cq, NULL);
+    grpc_cq_end_op(&exec_ctx, cq, NULL, GRPC_ERROR_NONE,
+                   DoneWithCompletionOnStack, NULL, &completion);
+    grpc_exec_ctx_finish(&exec_ctx);
+    grpc_completion_queue_pluck(cq, NULL, deadline, NULL);
+  }
+  grpc_completion_queue_destroy(cq);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_Pluck1Core);
+
+static void BM_EmptyCore(benchmark::State& state) {
+  TrackCounters track_counters;
+  grpc_completion_queue* cq = grpc_completion_queue_create(NULL);
+  gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC);
+  while (state.KeepRunning()) {
+    grpc_completion_queue_next(cq, deadline, NULL);
+  }
+  grpc_completion_queue_destroy(cq);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_EmptyCore);
+
 }  // namespace testing
 }  // namespace grpc
 
diff --git a/test/cpp/microbenchmarks/bm_error.cc b/test/cpp/microbenchmarks/bm_error.cc
index 8a4b86f281..c4f6aa19d5 100644
--- a/test/cpp/microbenchmarks/bm_error.cc
+++ b/test/cpp/microbenchmarks/bm_error.cc
@@ -40,8 +40,11 @@ extern "C" {
 #include "src/core/lib/transport/error_utils.h"
 }
 
+#include "test/cpp/microbenchmarks/helpers.h"
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
+auto& force_library_initialization = Library::get();
+
 class ErrorDeleter {
  public:
   void operator()(grpc_error* error) { GRPC_ERROR_UNREF(error); }
@@ -49,31 +52,74 @@ class ErrorDeleter {
 typedef std::unique_ptr<grpc_error, ErrorDeleter> ErrorPtr;
 
 static void BM_ErrorCreate(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     GRPC_ERROR_UNREF(GRPC_ERROR_CREATE("Error"));
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ErrorCreate);
 
 static void BM_ErrorCreateAndSetStatus(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     GRPC_ERROR_UNREF(grpc_error_set_int(GRPC_ERROR_CREATE("Error"),
                                         GRPC_ERROR_INT_GRPC_STATUS,
                                         GRPC_STATUS_ABORTED));
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ErrorCreateAndSetStatus);
 
+static void BM_ErrorCreateAndSetIntAndStr(benchmark::State& state) {
+  TrackCounters track_counters;
+  while (state.KeepRunning()) {
+    GRPC_ERROR_UNREF(grpc_error_set_str(
+        grpc_error_set_int(GRPC_ERROR_CREATE("GOAWAY received"),
+                           GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)0),
+        GRPC_ERROR_STR_RAW_BYTES, "raw bytes"));
+  }
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_ErrorCreateAndSetIntAndStr);
+
+static void BM_ErrorCreateAndSetIntLoop(benchmark::State& state) {
+  TrackCounters track_counters;
+  grpc_error* error = GRPC_ERROR_CREATE("Error");
+  int n = 0;
+  while (state.KeepRunning()) {
+    error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, n++);
+  }
+  GRPC_ERROR_UNREF(error);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_ErrorCreateAndSetIntLoop);
+
+static void BM_ErrorCreateAndSetStrLoop(benchmark::State& state) {
+  TrackCounters track_counters;
+  grpc_error* error = GRPC_ERROR_CREATE("Error");
+  const char* str = "hello";
+  while (state.KeepRunning()) {
+    error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, str);
+  }
+  GRPC_ERROR_UNREF(error);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_ErrorCreateAndSetStrLoop);
+
 static void BM_ErrorRefUnref(benchmark::State& state) {
+  TrackCounters track_counters;
   grpc_error* error = GRPC_ERROR_CREATE("Error");
   while (state.KeepRunning()) {
     GRPC_ERROR_UNREF(GRPC_ERROR_REF(error));
   }
   GRPC_ERROR_UNREF(error);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ErrorRefUnref);
 
 static void BM_ErrorUnrefNone(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     GRPC_ERROR_UNREF(GRPC_ERROR_NONE);
   }
@@ -81,30 +127,36 @@ static void BM_ErrorUnrefNone(benchmark::State& state) {
 BENCHMARK(BM_ErrorUnrefNone);
 
 static void BM_ErrorGetIntFromNoError(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     intptr_t value;
     grpc_error_get_int(GRPC_ERROR_NONE, GRPC_ERROR_INT_GRPC_STATUS, &value);
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ErrorGetIntFromNoError);
 
 static void BM_ErrorGetMissingInt(benchmark::State& state) {
+  TrackCounters track_counters;
   ErrorPtr error(
       grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_INDEX, 1));
   while (state.KeepRunning()) {
     intptr_t value;
     grpc_error_get_int(error.get(), GRPC_ERROR_INT_OFFSET, &value);
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ErrorGetMissingInt);
 
 static void BM_ErrorGetPresentInt(benchmark::State& state) {
+  TrackCounters track_counters;
   ErrorPtr error(
       grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_OFFSET, 1));
   while (state.KeepRunning()) {
     intptr_t value;
     grpc_error_get_int(error.get(), GRPC_ERROR_INT_OFFSET, &value);
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ErrorGetPresentInt);
 
@@ -177,22 +229,27 @@ class ErrorWithNestedGrpcStatus {
 
 template <class Fixture>
 static void BM_ErrorStringOnNewError(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     Fixture fixture;
     grpc_error_string(fixture.error());
   }
+  track_counters.Finish(state);
 }
 
 template <class Fixture>
 static void BM_ErrorStringRepeatedly(benchmark::State& state) {
+  TrackCounters track_counters;
   Fixture fixture;
   while (state.KeepRunning()) {
     grpc_error_string(fixture.error());
   }
+  track_counters.Finish(state);
 }
 
 template <class Fixture>
 static void BM_ErrorGetStatus(benchmark::State& state) {
+  TrackCounters track_counters;
   Fixture fixture;
   while (state.KeepRunning()) {
     grpc_status_code status;
@@ -200,34 +257,41 @@ static void BM_ErrorGetStatus(benchmark::State& state) {
     grpc_error_get_status(fixture.error(), fixture.deadline(), &status, &msg,
                           NULL);
   }
+  track_counters.Finish(state);
 }
 
 template <class Fixture>
 static void BM_ErrorGetStatusCode(benchmark::State& state) {
+  TrackCounters track_counters;
   Fixture fixture;
   while (state.KeepRunning()) {
     grpc_status_code status;
     grpc_error_get_status(fixture.error(), fixture.deadline(), &status, NULL,
                           NULL);
   }
+  track_counters.Finish(state);
 }
 
 template <class Fixture>
 static void BM_ErrorHttpError(benchmark::State& state) {
+  TrackCounters track_counters;
   Fixture fixture;
   while (state.KeepRunning()) {
     grpc_http2_error_code error;
     grpc_error_get_status(fixture.error(), fixture.deadline(), NULL, NULL,
                           &error);
   }
+  track_counters.Finish(state);
 }
 
 template <class Fixture>
 static void BM_HasClearGrpcStatus(benchmark::State& state) {
+  TrackCounters track_counters;
   Fixture fixture;
   while (state.KeepRunning()) {
     grpc_error_has_clear_grpc_status(fixture.error());
   }
+  track_counters.Finish(state);
 }
 
 #define BENCHMARK_SUITE(fixture)                         \
diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc
deleted file mode 100644
index 48e131f1be..0000000000
--- a/test/cpp/microbenchmarks/bm_fullstack.cc
+++ /dev/null
@@ -1,1079 +0,0 @@
-/*
- *
- * Copyright 2016, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/* Benchmark gRPC end2end in various configurations */
-
-#include <sstream>
-
-#include <grpc++/channel.h>
-#include <grpc++/create_channel.h>
-#include <grpc++/impl/grpc_library.h>
-#include <grpc++/security/credentials.h>
-#include <grpc++/security/server_credentials.h>
-#include <grpc++/server.h>
-#include <grpc++/server_builder.h>
-#include <grpc/support/log.h>
-
-extern "C" {
-#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
-#include "src/core/ext/transport/chttp2/transport/internal.h"
-#include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/iomgr/endpoint.h"
-#include "src/core/lib/iomgr/endpoint_pair.h"
-#include "src/core/lib/iomgr/exec_ctx.h"
-#include "src/core/lib/iomgr/tcp_posix.h"
-#include "src/core/lib/surface/channel.h"
-#include "src/core/lib/surface/completion_queue.h"
-#include "src/core/lib/surface/server.h"
-#include "test/core/util/memory_counters.h"
-#include "test/core/util/passthru_endpoint.h"
-#include "test/core/util/port.h"
-#include "test/core/util/trickle_endpoint.h"
-}
-#include "src/core/lib/profiling/timers.h"
-#include "src/cpp/client/create_channel_internal.h"
-#include "src/proto/grpc/testing/echo.grpc.pb.h"
-#include "third_party/benchmark/include/benchmark/benchmark.h"
-
-namespace grpc {
-namespace testing {
-
-static class InitializeStuff {
- public:
-  InitializeStuff() {
-    grpc_memory_counters_init();
-    init_lib_.init();
-    rq_ = grpc_resource_quota_create("bm");
-  }
-
-  ~InitializeStuff() { init_lib_.shutdown(); }
-
-  grpc_resource_quota* rq() { return rq_; }
-
- private:
-  internal::GrpcLibrary init_lib_;
-  grpc_resource_quota* rq_;
-} initialize_stuff;
-
-/*******************************************************************************
- * FIXTURES
- */
-
-static void ApplyCommonServerBuilderConfig(ServerBuilder* b) {
-  b->SetMaxReceiveMessageSize(INT_MAX);
-  b->SetMaxSendMessageSize(INT_MAX);
-}
-
-static void ApplyCommonChannelArguments(ChannelArguments* c) {
-  c->SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, INT_MAX);
-  c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX);
-}
-
-#ifdef GPR_LOW_LEVEL_COUNTERS
-extern "C" gpr_atm gpr_mu_locks;
-extern "C" gpr_atm gpr_counter_atm_cas;
-extern "C" gpr_atm gpr_counter_atm_add;
-#endif
-
-class BaseFixture {
- public:
-  void Finish(benchmark::State& s) {
-    std::ostringstream out;
-    this->AddToLabel(out, s);
-#ifdef GPR_LOW_LEVEL_COUNTERS
-    out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) -
-                                       mu_locks_at_start_) /
-                              (double)s.iterations())
-        << " atm_cas/iter:"
-        << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) -
-                     atm_cas_at_start_) /
-            (double)s.iterations())
-        << " atm_add/iter:"
-        << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) -
-                     atm_add_at_start_) /
-            (double)s.iterations());
-#endif
-    grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot();
-    out << " allocs/iter:"
-        << ((double)(counters_at_end.total_allocs_absolute -
-                     counters_at_start_.total_allocs_absolute) /
-            (double)s.iterations());
-    auto label = out.str();
-    if (label.length() && label[0] == ' ') {
-      label = label.substr(1);
-    }
-    s.SetLabel(label);
-  }
-
-  virtual void AddToLabel(std::ostream& out, benchmark::State& s) = 0;
-
- private:
-#ifdef GPR_LOW_LEVEL_COUNTERS
-  const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks);
-  const size_t atm_cas_at_start_ =
-      gpr_atm_no_barrier_load(&gpr_counter_atm_cas);
-  const size_t atm_add_at_start_ =
-      gpr_atm_no_barrier_load(&gpr_counter_atm_add);
-#endif
-  grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot();
-};
-
-class FullstackFixture : public BaseFixture {
- public:
-  FullstackFixture(Service* service, const grpc::string& address) {
-    ServerBuilder b;
-    b.AddListeningPort(address, InsecureServerCredentials());
-    cq_ = b.AddCompletionQueue(true);
-    b.RegisterService(service);
-    ApplyCommonServerBuilderConfig(&b);
-    server_ = b.BuildAndStart();
-    ChannelArguments args;
-    ApplyCommonChannelArguments(&args);
-    channel_ = CreateCustomChannel(address, InsecureChannelCredentials(), args);
-  }
-
-  virtual ~FullstackFixture() {
-    server_->Shutdown();
-    cq_->Shutdown();
-    void* tag;
-    bool ok;
-    while (cq_->Next(&tag, &ok)) {
-    }
-  }
-
-  ServerCompletionQueue* cq() { return cq_.get(); }
-  std::shared_ptr<Channel> channel() { return channel_; }
-
- private:
-  std::unique_ptr<Server> server_;
-  std::unique_ptr<ServerCompletionQueue> cq_;
-  std::shared_ptr<Channel> channel_;
-};
-
-class TCP : public FullstackFixture {
- public:
-  TCP(Service* service) : FullstackFixture(service, MakeAddress()) {}
-
-  void AddToLabel(std::ostream& out, benchmark::State& state) {}
-
- private:
-  static grpc::string MakeAddress() {
-    int port = grpc_pick_unused_port_or_die();
-    std::stringstream addr;
-    addr << "localhost:" << port;
-    return addr.str();
-  }
-};
-
-class UDS : public FullstackFixture {
- public:
-  UDS(Service* service) : FullstackFixture(service, MakeAddress()) {}
-
-  void AddToLabel(std::ostream& out, benchmark::State& state) override {}
-
- private:
-  static grpc::string MakeAddress() {
-    int port = grpc_pick_unused_port_or_die();  // just for a unique id - not a
-                                                // real port
-    std::stringstream addr;
-    addr << "unix:/tmp/bm_fullstack." << port;
-    return addr.str();
-  }
-};
-
-class EndpointPairFixture : public BaseFixture {
- public:
-  EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints)
-      : endpoint_pair_(endpoints) {
-    ServerBuilder b;
-    cq_ = b.AddCompletionQueue(true);
-    b.RegisterService(service);
-    ApplyCommonServerBuilderConfig(&b);
-    server_ = b.BuildAndStart();
-
-    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-
-    /* add server endpoint to server_ */
-    {
-      const grpc_channel_args* server_args =
-          grpc_server_get_channel_args(server_->c_server());
-      server_transport_ = grpc_create_chttp2_transport(
-          &exec_ctx, server_args, endpoints.server, 0 /* is_client */);
-
-      grpc_pollset** pollsets;
-      size_t num_pollsets = 0;
-      grpc_server_get_pollsets(server_->c_server(), &pollsets, &num_pollsets);
-
-      for (size_t i = 0; i < num_pollsets; i++) {
-        grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]);
-      }
-
-      grpc_server_setup_transport(&exec_ctx, server_->c_server(),
-                                  server_transport_, NULL, server_args);
-      grpc_chttp2_transport_start_reading(&exec_ctx, server_transport_, NULL);
-    }
-
-    /* create channel */
-    {
-      ChannelArguments args;
-      args.SetString(GRPC_ARG_DEFAULT_AUTHORITY, "test.authority");
-      ApplyCommonChannelArguments(&args);
-
-      grpc_channel_args c_args = args.c_channel_args();
-      client_transport_ =
-          grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1);
-      GPR_ASSERT(client_transport_);
-      grpc_channel* channel =
-          grpc_channel_create(&exec_ctx, "target", &c_args,
-                              GRPC_CLIENT_DIRECT_CHANNEL, client_transport_);
-      grpc_chttp2_transport_start_reading(&exec_ctx, client_transport_, NULL);
-
-      channel_ = CreateChannelInternal("", channel);
-    }
-
-    grpc_exec_ctx_finish(&exec_ctx);
-  }
-
-  virtual ~EndpointPairFixture() {
-    server_->Shutdown();
-    cq_->Shutdown();
-    void* tag;
-    bool ok;
-    while (cq_->Next(&tag, &ok)) {
-    }
-  }
-
-  ServerCompletionQueue* cq() { return cq_.get(); }
-  std::shared_ptr<Channel> channel() { return channel_; }
-
- protected:
-  grpc_endpoint_pair endpoint_pair_;
-  grpc_transport* client_transport_;
-  grpc_transport* server_transport_;
-
- private:
-  std::unique_ptr<Server> server_;
-  std::unique_ptr<ServerCompletionQueue> cq_;
-  std::shared_ptr<Channel> channel_;
-};
-
-class SockPair : public EndpointPairFixture {
- public:
-  SockPair(Service* service)
-      : EndpointPairFixture(service, grpc_iomgr_create_endpoint_pair(
-                                         "test", initialize_stuff.rq(), 8192)) {
-  }
-
-  void AddToLabel(std::ostream& out, benchmark::State& state) {}
-};
-
-class InProcessCHTTP2 : public EndpointPairFixture {
- public:
-  InProcessCHTTP2(Service* service)
-      : EndpointPairFixture(service, MakeEndpoints()) {}
-
-  void AddToLabel(std::ostream& out, benchmark::State& state) {
-    out << " writes/iter:"
-        << ((double)stats_.num_writes / (double)state.iterations());
-  }
-
- private:
-  grpc_passthru_endpoint_stats stats_;
-
-  grpc_endpoint_pair MakeEndpoints() {
-    grpc_endpoint_pair p;
-    grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(),
-                                  &stats_);
-    return p;
-  }
-};
-
-class TrickledCHTTP2 : public EndpointPairFixture {
- public:
-  TrickledCHTTP2(Service* service, size_t megabits_per_second)
-      : EndpointPairFixture(service, MakeEndpoints(megabits_per_second)) {}
-
-  void AddToLabel(std::ostream& out, benchmark::State& state) {
-    out << " writes/iter:"
-        << ((double)stats_.num_writes / (double)state.iterations())
-        << " cli_transport_stalls/iter:"
-        << ((double)
-                client_stats_.streams_stalled_due_to_transport_flow_control /
-            (double)state.iterations())
-        << " cli_stream_stalls/iter:"
-        << ((double)client_stats_.streams_stalled_due_to_stream_flow_control /
-            (double)state.iterations())
-        << " svr_transport_stalls/iter:"
-        << ((double)
-                server_stats_.streams_stalled_due_to_transport_flow_control /
-            (double)state.iterations())
-        << " svr_stream_stalls/iter:"
-        << ((double)server_stats_.streams_stalled_due_to_stream_flow_control /
-            (double)state.iterations());
-  }
-
-  void Step() {
-    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-    size_t client_backlog =
-        grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.client);
-    size_t server_backlog =
-        grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.server);
-    grpc_exec_ctx_finish(&exec_ctx);
-
-    UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_,
-                client_backlog);
-    UpdateStats((grpc_chttp2_transport*)server_transport_, &server_stats_,
-                server_backlog);
-  }
-
- private:
-  grpc_passthru_endpoint_stats stats_;
-  struct Stats {
-    int streams_stalled_due_to_stream_flow_control = 0;
-    int streams_stalled_due_to_transport_flow_control = 0;
-  };
-  Stats client_stats_;
-  Stats server_stats_;
-
-  grpc_endpoint_pair MakeEndpoints(size_t kilobits) {
-    grpc_endpoint_pair p;
-    grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(),
-                                  &stats_);
-    double bytes_per_second = 125.0 * kilobits;
-    p.client = grpc_trickle_endpoint_create(p.client, bytes_per_second);
-    p.server = grpc_trickle_endpoint_create(p.server, bytes_per_second);
-    return p;
-  }
-
-  void UpdateStats(grpc_chttp2_transport* t, Stats* s, size_t backlog) {
-    if (backlog == 0) {
-      if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_STREAM].head != NULL) {
-        s->streams_stalled_due_to_stream_flow_control++;
-      }
-      if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT].head != NULL) {
-        s->streams_stalled_due_to_transport_flow_control++;
-      }
-    }
-  }
-};
-
-/*******************************************************************************
- * CONTEXT MUTATORS
- */
-
-static const int kPregenerateKeyCount = 100000;
-
-template <class F>
-auto MakeVector(size_t length, F f) -> std::vector<decltype(f())> {
-  std::vector<decltype(f())> out;
-  out.reserve(length);
-  for (size_t i = 0; i < length; i++) {
-    out.push_back(f());
-  }
-  return out;
-}
-
-class NoOpMutator {
- public:
-  template <class ContextType>
-  NoOpMutator(ContextType* context) {}
-};
-
-template <int length>
-class RandomBinaryMetadata {
- public:
-  static const grpc::string& Key() { return kKey; }
-
-  static const grpc::string& Value() {
-    return kValues[rand() % kValues.size()];
-  }
-
- private:
-  static const grpc::string kKey;
-  static const std::vector<grpc::string> kValues;
-
-  static grpc::string GenerateOneString() {
-    grpc::string s;
-    s.reserve(length + 1);
-    for (int i = 0; i < length; i++) {
-      s += (char)rand();
-    }
-    return s;
-  }
-};
-
-template <int length>
-const grpc::string RandomBinaryMetadata<length>::kKey = "foo-bin";
-
-template <int length>
-const std::vector<grpc::string> RandomBinaryMetadata<length>::kValues =
-    MakeVector(kPregenerateKeyCount, GenerateOneString);
-
-template <int length>
-class RandomAsciiMetadata {
- public:
-  static const grpc::string& Key() { return kKey; }
-
-  static const grpc::string& Value() {
-    return kValues[rand() % kValues.size()];
-  }
-
- private:
-  static const grpc::string kKey;
-  static const std::vector<grpc::string> kValues;
-
-  static grpc::string GenerateOneString() {
-    grpc::string s;
-    s.reserve(length + 1);
-    for (int i = 0; i < length; i++) {
-      s += (char)(rand() % 26 + 'a');
-    }
-    return s;
-  }
-};
-
-template <int length>
-const grpc::string RandomAsciiMetadata<length>::kKey = "foo";
-
-template <int length>
-const std::vector<grpc::string> RandomAsciiMetadata<length>::kValues =
-    MakeVector(kPregenerateKeyCount, GenerateOneString);
-
-template <class Generator, int kNumKeys>
-class Client_AddMetadata : public NoOpMutator {
- public:
-  Client_AddMetadata(ClientContext* context) : NoOpMutator(context) {
-    for (int i = 0; i < kNumKeys; i++) {
-      context->AddMetadata(Generator::Key(), Generator::Value());
-    }
-  }
-};
-
-template <class Generator, int kNumKeys>
-class Server_AddInitialMetadata : public NoOpMutator {
- public:
-  Server_AddInitialMetadata(ServerContext* context) : NoOpMutator(context) {
-    for (int i = 0; i < kNumKeys; i++) {
-      context->AddInitialMetadata(Generator::Key(), Generator::Value());
-    }
-  }
-};
-
-/*******************************************************************************
- * BENCHMARKING KERNELS
- */
-
-static void* tag(intptr_t x) { return reinterpret_cast<void*>(x); }
-
-template <class Fixture, class ClientContextMutator, class ServerContextMutator>
-static void BM_UnaryPingPong(benchmark::State& state) {
-  EchoTestService::AsyncService service;
-  std::unique_ptr<Fixture> fixture(new Fixture(&service));
-  EchoRequest send_request;
-  EchoResponse send_response;
-  EchoResponse recv_response;
-  if (state.range(0) > 0) {
-    send_request.set_message(std::string(state.range(0), 'a'));
-  }
-  if (state.range(1) > 0) {
-    send_response.set_message(std::string(state.range(1), 'a'));
-  }
-  Status recv_status;
-  struct ServerEnv {
-    ServerContext ctx;
-    EchoRequest recv_request;
-    grpc::ServerAsyncResponseWriter<EchoResponse> response_writer;
-    ServerEnv() : response_writer(&ctx) {}
-  };
-  uint8_t server_env_buffer[2 * sizeof(ServerEnv)];
-  ServerEnv* server_env[2] = {
-      reinterpret_cast<ServerEnv*>(server_env_buffer),
-      reinterpret_cast<ServerEnv*>(server_env_buffer + sizeof(ServerEnv))};
-  new (server_env[0]) ServerEnv;
-  new (server_env[1]) ServerEnv;
-  service.RequestEcho(&server_env[0]->ctx, &server_env[0]->recv_request,
-                      &server_env[0]->response_writer, fixture->cq(),
-                      fixture->cq(), tag(0));
-  service.RequestEcho(&server_env[1]->ctx, &server_env[1]->recv_request,
-                      &server_env[1]->response_writer, fixture->cq(),
-                      fixture->cq(), tag(1));
-  std::unique_ptr<EchoTestService::Stub> stub(
-      EchoTestService::NewStub(fixture->channel()));
-  while (state.KeepRunning()) {
-    GPR_TIMER_SCOPE("BenchmarkCycle", 0);
-    recv_response.Clear();
-    ClientContext cli_ctx;
-    ClientContextMutator cli_ctx_mut(&cli_ctx);
-    std::unique_ptr<ClientAsyncResponseReader<EchoResponse>> response_reader(
-        stub->AsyncEcho(&cli_ctx, send_request, fixture->cq()));
-    void* t;
-    bool ok;
-    GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-    GPR_ASSERT(ok);
-    GPR_ASSERT(t == tag(0) || t == tag(1));
-    intptr_t slot = reinterpret_cast<intptr_t>(t);
-    ServerEnv* senv = server_env[slot];
-    ServerContextMutator svr_ctx_mut(&senv->ctx);
-    senv->response_writer.Finish(send_response, Status::OK, tag(3));
-    response_reader->Finish(&recv_response, &recv_status, tag(4));
-    for (int i = (1 << 3) | (1 << 4); i != 0;) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      GPR_ASSERT(ok);
-      int tagnum = (int)reinterpret_cast<intptr_t>(t);
-      GPR_ASSERT(i & (1 << tagnum));
-      i -= 1 << tagnum;
-    }
-    GPR_ASSERT(recv_status.ok());
-
-    senv->~ServerEnv();
-    senv = new (senv) ServerEnv();
-    service.RequestEcho(&senv->ctx, &senv->recv_request, &senv->response_writer,
-                        fixture->cq(), fixture->cq(), tag(slot));
-  }
-  fixture->Finish(state);
-  fixture.reset();
-  server_env[0]->~ServerEnv();
-  server_env[1]->~ServerEnv();
-  state.SetBytesProcessed(state.range(0) * state.iterations() +
-                          state.range(1) * state.iterations());
-}
-
-// Repeatedly makes Streaming Bidi calls (exchanging a configurable number of
-// messages in each call) in a loop on a single channel
-//
-//  First parmeter (i.e state.range(0)):  Message size (in bytes) to use
-//  Second parameter (i.e state.range(1)): Number of ping pong messages.
-//      Note: One ping-pong means two messages (one from client to server and
-//      the other from server to client):
-template <class Fixture, class ClientContextMutator, class ServerContextMutator>
-static void BM_StreamingPingPong(benchmark::State& state) {
-  const int msg_size = state.range(0);
-  const int max_ping_pongs = state.range(1);
-
-  EchoTestService::AsyncService service;
-  std::unique_ptr<Fixture> fixture(new Fixture(&service));
-  {
-    EchoResponse send_response;
-    EchoResponse recv_response;
-    EchoRequest send_request;
-    EchoRequest recv_request;
-
-    if (msg_size > 0) {
-      send_request.set_message(std::string(msg_size, 'a'));
-      send_response.set_message(std::string(msg_size, 'b'));
-    }
-
-    std::unique_ptr<EchoTestService::Stub> stub(
-        EchoTestService::NewStub(fixture->channel()));
-
-    while (state.KeepRunning()) {
-      ServerContext svr_ctx;
-      ServerContextMutator svr_ctx_mut(&svr_ctx);
-      ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
-      service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
-                                fixture->cq(), tag(0));
-
-      ClientContext cli_ctx;
-      ClientContextMutator cli_ctx_mut(&cli_ctx);
-      auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
-
-      // Establish async stream between client side and server side
-      void* t;
-      bool ok;
-      int need_tags = (1 << 0) | (1 << 1);
-      while (need_tags) {
-        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-        GPR_ASSERT(ok);
-        int i = (int)(intptr_t)t;
-        GPR_ASSERT(need_tags & (1 << i));
-        need_tags &= ~(1 << i);
-      }
-
-      // Send 'max_ping_pongs' number of ping pong messages
-      int ping_pong_cnt = 0;
-      while (ping_pong_cnt < max_ping_pongs) {
-        request_rw->Write(send_request, tag(0));   // Start client send
-        response_rw.Read(&recv_request, tag(1));   // Start server recv
-        request_rw->Read(&recv_response, tag(2));  // Start client recv
-
-        need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3);
-        while (need_tags) {
-          GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-          GPR_ASSERT(ok);
-          int i = (int)(intptr_t)t;
-
-          // If server recv is complete, start the server send operation
-          if (i == 1) {
-            response_rw.Write(send_response, tag(3));
-          }
-
-          GPR_ASSERT(need_tags & (1 << i));
-          need_tags &= ~(1 << i);
-        }
-
-        ping_pong_cnt++;
-      }
-
-      request_rw->WritesDone(tag(0));
-      response_rw.Finish(Status::OK, tag(1));
-
-      Status recv_status;
-      request_rw->Finish(&recv_status, tag(2));
-
-      need_tags = (1 << 0) | (1 << 1) | (1 << 2);
-      while (need_tags) {
-        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-        int i = (int)(intptr_t)t;
-        GPR_ASSERT(need_tags & (1 << i));
-        need_tags &= ~(1 << i);
-      }
-
-      GPR_ASSERT(recv_status.ok());
-    }
-  }
-
-  fixture->Finish(state);
-  fixture.reset();
-  state.SetBytesProcessed(msg_size * state.iterations() * max_ping_pongs * 2);
-}
-
-// Repeatedly sends ping pong messages in a single streaming Bidi call in a loop
-//     First parmeter (i.e state.range(0)):  Message size (in bytes) to use
-template <class Fixture, class ClientContextMutator, class ServerContextMutator>
-static void BM_StreamingPingPongMsgs(benchmark::State& state) {
-  const int msg_size = state.range(0);
-
-  EchoTestService::AsyncService service;
-  std::unique_ptr<Fixture> fixture(new Fixture(&service));
-  {
-    EchoResponse send_response;
-    EchoResponse recv_response;
-    EchoRequest send_request;
-    EchoRequest recv_request;
-
-    if (msg_size > 0) {
-      send_request.set_message(std::string(msg_size, 'a'));
-      send_response.set_message(std::string(msg_size, 'b'));
-    }
-
-    std::unique_ptr<EchoTestService::Stub> stub(
-        EchoTestService::NewStub(fixture->channel()));
-
-    ServerContext svr_ctx;
-    ServerContextMutator svr_ctx_mut(&svr_ctx);
-    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
-    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
-                              fixture->cq(), tag(0));
-
-    ClientContext cli_ctx;
-    ClientContextMutator cli_ctx_mut(&cli_ctx);
-    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
-
-    // Establish async stream between client side and server side
-    void* t;
-    bool ok;
-    int need_tags = (1 << 0) | (1 << 1);
-    while (need_tags) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      GPR_ASSERT(ok);
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-
-    while (state.KeepRunning()) {
-      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
-      request_rw->Write(send_request, tag(0));   // Start client send
-      response_rw.Read(&recv_request, tag(1));   // Start server recv
-      request_rw->Read(&recv_response, tag(2));  // Start client recv
-
-      need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3);
-      while (need_tags) {
-        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-        GPR_ASSERT(ok);
-        int i = (int)(intptr_t)t;
-
-        // If server recv is complete, start the server send operation
-        if (i == 1) {
-          response_rw.Write(send_response, tag(3));
-        }
-
-        GPR_ASSERT(need_tags & (1 << i));
-        need_tags &= ~(1 << i);
-      }
-    }
-
-    request_rw->WritesDone(tag(0));
-    response_rw.Finish(Status::OK, tag(1));
-    Status recv_status;
-    request_rw->Finish(&recv_status, tag(2));
-
-    need_tags = (1 << 0) | (1 << 1) | (1 << 2);
-    while (need_tags) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-
-    GPR_ASSERT(recv_status.ok());
-  }
-
-  fixture->Finish(state);
-  fixture.reset();
-  state.SetBytesProcessed(msg_size * state.iterations() * 2);
-}
-
-template <class Fixture>
-static void BM_PumpStreamClientToServer(benchmark::State& state) {
-  EchoTestService::AsyncService service;
-  std::unique_ptr<Fixture> fixture(new Fixture(&service));
-  {
-    EchoRequest send_request;
-    EchoRequest recv_request;
-    if (state.range(0) > 0) {
-      send_request.set_message(std::string(state.range(0), 'a'));
-    }
-    Status recv_status;
-    ServerContext svr_ctx;
-    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
-    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
-                              fixture->cq(), tag(0));
-    std::unique_ptr<EchoTestService::Stub> stub(
-        EchoTestService::NewStub(fixture->channel()));
-    ClientContext cli_ctx;
-    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
-    int need_tags = (1 << 0) | (1 << 1);
-    void* t;
-    bool ok;
-    while (need_tags) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      GPR_ASSERT(ok);
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-    response_rw.Read(&recv_request, tag(0));
-    while (state.KeepRunning()) {
-      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
-      request_rw->Write(send_request, tag(1));
-      while (true) {
-        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-        if (t == tag(0)) {
-          response_rw.Read(&recv_request, tag(0));
-        } else if (t == tag(1)) {
-          break;
-        } else {
-          GPR_ASSERT(false);
-        }
-      }
-    }
-    request_rw->WritesDone(tag(1));
-    need_tags = (1 << 0) | (1 << 1);
-    while (need_tags) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-  }
-  fixture->Finish(state);
-  fixture.reset();
-  state.SetBytesProcessed(state.range(0) * state.iterations());
-}
-
-template <class Fixture>
-static void BM_PumpStreamServerToClient(benchmark::State& state) {
-  EchoTestService::AsyncService service;
-  std::unique_ptr<Fixture> fixture(new Fixture(&service));
-  {
-    EchoResponse send_response;
-    EchoResponse recv_response;
-    if (state.range(0) > 0) {
-      send_response.set_message(std::string(state.range(0), 'a'));
-    }
-    Status recv_status;
-    ServerContext svr_ctx;
-    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
-    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
-                              fixture->cq(), tag(0));
-    std::unique_ptr<EchoTestService::Stub> stub(
-        EchoTestService::NewStub(fixture->channel()));
-    ClientContext cli_ctx;
-    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
-    int need_tags = (1 << 0) | (1 << 1);
-    void* t;
-    bool ok;
-    while (need_tags) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      GPR_ASSERT(ok);
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-    request_rw->Read(&recv_response, tag(0));
-    while (state.KeepRunning()) {
-      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
-      response_rw.Write(send_response, tag(1));
-      while (true) {
-        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-        if (t == tag(0)) {
-          request_rw->Read(&recv_response, tag(0));
-        } else if (t == tag(1)) {
-          break;
-        } else {
-          GPR_ASSERT(false);
-        }
-      }
-    }
-    response_rw.Finish(Status::OK, tag(1));
-    need_tags = (1 << 0) | (1 << 1);
-    while (need_tags) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-  }
-  fixture->Finish(state);
-  fixture.reset();
-  state.SetBytesProcessed(state.range(0) * state.iterations());
-}
-
-static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok) {
-  while (true) {
-    switch (fixture->cq()->AsyncNext(
-        t, ok, gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
-                            gpr_time_from_micros(100, GPR_TIMESPAN)))) {
-      case CompletionQueue::TIMEOUT:
-        fixture->Step();
-        break;
-      case CompletionQueue::SHUTDOWN:
-        GPR_ASSERT(false);
-        break;
-      case CompletionQueue::GOT_EVENT:
-        return;
-    }
-  }
-}
-
-static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) {
-  EchoTestService::AsyncService service;
-  std::unique_ptr<TrickledCHTTP2> fixture(
-      new TrickledCHTTP2(&service, state.range(1)));
-  {
-    EchoResponse send_response;
-    EchoResponse recv_response;
-    if (state.range(0) > 0) {
-      send_response.set_message(std::string(state.range(0), 'a'));
-    }
-    Status recv_status;
-    ServerContext svr_ctx;
-    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
-    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
-                              fixture->cq(), tag(0));
-    std::unique_ptr<EchoTestService::Stub> stub(
-        EchoTestService::NewStub(fixture->channel()));
-    ClientContext cli_ctx;
-    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
-    int need_tags = (1 << 0) | (1 << 1);
-    void* t;
-    bool ok;
-    while (need_tags) {
-      TrickleCQNext(fixture.get(), &t, &ok);
-      GPR_ASSERT(ok);
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-    request_rw->Read(&recv_response, tag(0));
-    while (state.KeepRunning()) {
-      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
-      response_rw.Write(send_response, tag(1));
-      while (true) {
-        TrickleCQNext(fixture.get(), &t, &ok);
-        if (t == tag(0)) {
-          request_rw->Read(&recv_response, tag(0));
-        } else if (t == tag(1)) {
-          break;
-        } else {
-          GPR_ASSERT(false);
-        }
-      }
-    }
-    response_rw.Finish(Status::OK, tag(1));
-    need_tags = (1 << 0) | (1 << 1);
-    while (need_tags) {
-      TrickleCQNext(fixture.get(), &t, &ok);
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-  }
-  fixture->Finish(state);
-  fixture.reset();
-  state.SetBytesProcessed(state.range(0) * state.iterations());
-}
-
-/*******************************************************************************
- * CONFIGURATIONS
- */
-
-static void SweepSizesArgs(benchmark::internal::Benchmark* b) {
-  b->Args({0, 0});
-  for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) {
-    b->Args({i, 0});
-    b->Args({0, i});
-    b->Args({i, i});
-  }
-}
-
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, TCP, NoOpMutator, NoOpMutator)
-    ->Apply(SweepSizesArgs);
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, UDS, NoOpMutator, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, SockPair, NoOpMutator, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, NoOpMutator)
-    ->Apply(SweepSizesArgs);
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomBinaryMetadata<10>, 1>, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomBinaryMetadata<31>, 1>, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomBinaryMetadata<100>, 1>,
-                   NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomBinaryMetadata<10>, 2>, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomBinaryMetadata<31>, 2>, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomBinaryMetadata<100>, 2>,
-                   NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
-                   Server_AddInitialMetadata<RandomBinaryMetadata<10>, 1>)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
-                   Server_AddInitialMetadata<RandomBinaryMetadata<31>, 1>)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
-                   Server_AddInitialMetadata<RandomBinaryMetadata<100>, 1>)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomAsciiMetadata<10>, 1>, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomAsciiMetadata<31>, 1>, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomAsciiMetadata<100>, 1>, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
-                   Server_AddInitialMetadata<RandomAsciiMetadata<10>, 1>)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
-                   Server_AddInitialMetadata<RandomAsciiMetadata<31>, 1>)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
-                   Server_AddInitialMetadata<RandomAsciiMetadata<100>, 1>)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
-                   Server_AddInitialMetadata<RandomAsciiMetadata<10>, 100>)
-    ->Args({0, 0});
-
-BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2)
-    ->Range(0, 128 * 1024 * 1024);
-
-static void TrickleArgs(benchmark::internal::Benchmark* b) {
-  for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) {
-    for (int j = 1; j <= 128 * 1024 * 1024; j *= 8) {
-      double expected_time =
-          static_cast<double>(14 + i) / (125.0 * static_cast<double>(j));
-      if (expected_time > 0.01) continue;
-      b->Args({i, j});
-    }
-  }
-}
-
-BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs);
-
-// Generate Args for StreamingPingPong benchmarks. Currently generates args for
-// only "small streams" (i.e streams with 0, 1 or 2 messages)
-static void StreamingPingPongArgs(benchmark::internal::Benchmark* b) {
-  int msg_size = 0;
-
-  b->Args({0, 0});  // spl case: 0 ping-pong msgs (msg_size doesn't matter here)
-
-  for (msg_size = 0; msg_size <= 128 * 1024 * 1024;
-       msg_size == 0 ? msg_size++ : msg_size *= 8) {
-    b->Args({msg_size, 1});
-    b->Args({msg_size, 2});
-  }
-}
-
-BENCHMARK_TEMPLATE(BM_StreamingPingPong, InProcessCHTTP2, NoOpMutator,
-                   NoOpMutator)
-    ->Apply(StreamingPingPongArgs);
-BENCHMARK_TEMPLATE(BM_StreamingPingPong, TCP, NoOpMutator, NoOpMutator)
-    ->Apply(StreamingPingPongArgs);
-
-BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, InProcessCHTTP2, NoOpMutator,
-                   NoOpMutator)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, TCP, NoOpMutator, NoOpMutator)
-    ->Range(0, 128 * 1024 * 1024);
-
-}  // namespace testing
-}  // namespace grpc
-
-BENCHMARK_MAIN();
diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc
new file mode 100644
index 0000000000..513974fde1
--- /dev/null
+++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc
@@ -0,0 +1,280 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* Benchmark gRPC end2end in various configurations */
+
+#include <sstream>
+
+#include "src/core/lib/profiling/timers.h"
+#include "src/cpp/client/create_channel_internal.h"
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
+#include "test/cpp/microbenchmarks/fullstack_context_mutators.h"
+#include "test/cpp/microbenchmarks/fullstack_fixtures.h"
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+
+namespace grpc {
+namespace testing {
+
+// force library initialization
+auto& force_library_initialization = Library::get();
+
+/*******************************************************************************
+ * BENCHMARKING KERNELS
+ */
+
+static void* tag(intptr_t x) { return reinterpret_cast<void*>(x); }
+
+// Repeatedly makes Streaming Bidi calls (exchanging a configurable number of
+// messages in each call) in a loop on a single channel
+//
+//  First parmeter (i.e state.range(0)):  Message size (in bytes) to use
+//  Second parameter (i.e state.range(1)): Number of ping pong messages.
+//      Note: One ping-pong means two messages (one from client to server and
+//      the other from server to client):
+template <class Fixture, class ClientContextMutator, class ServerContextMutator>
+static void BM_StreamingPingPong(benchmark::State& state) {
+  TrackCounters track_counters;
+  const int msg_size = state.range(0);
+  const int max_ping_pongs = state.range(1);
+
+  EchoTestService::AsyncService service;
+  std::unique_ptr<Fixture> fixture(new Fixture(&service));
+  {
+    EchoResponse send_response;
+    EchoResponse recv_response;
+    EchoRequest send_request;
+    EchoRequest recv_request;
+
+    if (msg_size > 0) {
+      send_request.set_message(std::string(msg_size, 'a'));
+      send_response.set_message(std::string(msg_size, 'b'));
+    }
+
+    std::unique_ptr<EchoTestService::Stub> stub(
+        EchoTestService::NewStub(fixture->channel()));
+
+    while (state.KeepRunning()) {
+      ServerContext svr_ctx;
+      ServerContextMutator svr_ctx_mut(&svr_ctx);
+      ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
+      service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
+                                fixture->cq(), tag(0));
+
+      ClientContext cli_ctx;
+      ClientContextMutator cli_ctx_mut(&cli_ctx);
+      auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
+
+      // Establish async stream between client side and server side
+      void* t;
+      bool ok;
+      int need_tags = (1 << 0) | (1 << 1);
+      while (need_tags) {
+        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+        GPR_ASSERT(ok);
+        int i = (int)(intptr_t)t;
+        GPR_ASSERT(need_tags & (1 << i));
+        need_tags &= ~(1 << i);
+      }
+
+      // Send 'max_ping_pongs' number of ping pong messages
+      int ping_pong_cnt = 0;
+      while (ping_pong_cnt < max_ping_pongs) {
+        request_rw->Write(send_request, tag(0));   // Start client send
+        response_rw.Read(&recv_request, tag(1));   // Start server recv
+        request_rw->Read(&recv_response, tag(2));  // Start client recv
+
+        need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3);
+        while (need_tags) {
+          GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+          GPR_ASSERT(ok);
+          int i = (int)(intptr_t)t;
+
+          // If server recv is complete, start the server send operation
+          if (i == 1) {
+            response_rw.Write(send_response, tag(3));
+          }
+
+          GPR_ASSERT(need_tags & (1 << i));
+          need_tags &= ~(1 << i);
+        }
+
+        ping_pong_cnt++;
+      }
+
+      request_rw->WritesDone(tag(0));
+      response_rw.Finish(Status::OK, tag(1));
+
+      Status recv_status;
+      request_rw->Finish(&recv_status, tag(2));
+
+      need_tags = (1 << 0) | (1 << 1) | (1 << 2);
+      while (need_tags) {
+        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+        int i = (int)(intptr_t)t;
+        GPR_ASSERT(need_tags & (1 << i));
+        need_tags &= ~(1 << i);
+      }
+
+      GPR_ASSERT(recv_status.ok());
+    }
+  }
+
+  fixture->Finish(state);
+  fixture.reset();
+  state.SetBytesProcessed(msg_size * state.iterations() * max_ping_pongs * 2);
+  track_counters.Finish(state);
+}
+
+// Repeatedly sends ping pong messages in a single streaming Bidi call in a loop
+//     First parmeter (i.e state.range(0)):  Message size (in bytes) to use
+template <class Fixture, class ClientContextMutator, class ServerContextMutator>
+static void BM_StreamingPingPongMsgs(benchmark::State& state) {
+  TrackCounters track_counters;
+  const int msg_size = state.range(0);
+
+  EchoTestService::AsyncService service;
+  std::unique_ptr<Fixture> fixture(new Fixture(&service));
+  {
+    EchoResponse send_response;
+    EchoResponse recv_response;
+    EchoRequest send_request;
+    EchoRequest recv_request;
+
+    if (msg_size > 0) {
+      send_request.set_message(std::string(msg_size, 'a'));
+      send_response.set_message(std::string(msg_size, 'b'));
+    }
+
+    std::unique_ptr<EchoTestService::Stub> stub(
+        EchoTestService::NewStub(fixture->channel()));
+
+    ServerContext svr_ctx;
+    ServerContextMutator svr_ctx_mut(&svr_ctx);
+    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
+    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
+                              fixture->cq(), tag(0));
+
+    ClientContext cli_ctx;
+    ClientContextMutator cli_ctx_mut(&cli_ctx);
+    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
+
+    // Establish async stream between client side and server side
+    void* t;
+    bool ok;
+    int need_tags = (1 << 0) | (1 << 1);
+    while (need_tags) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      GPR_ASSERT(ok);
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+
+    while (state.KeepRunning()) {
+      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
+      request_rw->Write(send_request, tag(0));   // Start client send
+      response_rw.Read(&recv_request, tag(1));   // Start server recv
+      request_rw->Read(&recv_response, tag(2));  // Start client recv
+
+      need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3);
+      while (need_tags) {
+        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+        GPR_ASSERT(ok);
+        int i = (int)(intptr_t)t;
+
+        // If server recv is complete, start the server send operation
+        if (i == 1) {
+          response_rw.Write(send_response, tag(3));
+        }
+
+        GPR_ASSERT(need_tags & (1 << i));
+        need_tags &= ~(1 << i);
+      }
+    }
+
+    request_rw->WritesDone(tag(0));
+    response_rw.Finish(Status::OK, tag(1));
+    Status recv_status;
+    request_rw->Finish(&recv_status, tag(2));
+
+    need_tags = (1 << 0) | (1 << 1) | (1 << 2);
+    while (need_tags) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+
+    GPR_ASSERT(recv_status.ok());
+  }
+
+  fixture->Finish(state);
+  fixture.reset();
+  state.SetBytesProcessed(msg_size * state.iterations() * 2);
+  track_counters.Finish(state);
+}
+
+/*******************************************************************************
+ * CONFIGURATIONS
+ */
+
+// Generate Args for StreamingPingPong benchmarks. Currently generates args for
+// only "small streams" (i.e streams with 0, 1 or 2 messages)
+static void StreamingPingPongArgs(benchmark::internal::Benchmark* b) {
+  int msg_size = 0;
+
+  b->Args({0, 0});  // spl case: 0 ping-pong msgs (msg_size doesn't matter here)
+
+  for (msg_size = 0; msg_size <= 128 * 1024 * 1024;
+       msg_size == 0 ? msg_size++ : msg_size *= 8) {
+    b->Args({msg_size, 1});
+    b->Args({msg_size, 2});
+  }
+}
+
+BENCHMARK_TEMPLATE(BM_StreamingPingPong, InProcessCHTTP2, NoOpMutator,
+                   NoOpMutator)
+    ->Apply(StreamingPingPongArgs);
+BENCHMARK_TEMPLATE(BM_StreamingPingPong, TCP, NoOpMutator, NoOpMutator)
+    ->Apply(StreamingPingPongArgs);
+
+BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, InProcessCHTTP2, NoOpMutator,
+                   NoOpMutator)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, TCP, NoOpMutator, NoOpMutator)
+    ->Range(0, 128 * 1024 * 1024);
+
+}  // namespace testing
+}  // namespace grpc
+
+BENCHMARK_MAIN();
diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc
new file mode 100644
index 0000000000..dc0e7d769a
--- /dev/null
+++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc
@@ -0,0 +1,197 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* Benchmark gRPC end2end in various configurations */
+
+#include <sstream>
+
+#include "src/core/lib/profiling/timers.h"
+#include "src/cpp/client/create_channel_internal.h"
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
+#include "test/cpp/microbenchmarks/fullstack_context_mutators.h"
+#include "test/cpp/microbenchmarks/fullstack_fixtures.h"
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+
+namespace grpc {
+namespace testing {
+
+// force library initialization
+auto& force_library_initialization = Library::get();
+
+/*******************************************************************************
+ * BENCHMARKING KERNELS
+ */
+
+static void* tag(intptr_t x) { return reinterpret_cast<void*>(x); }
+
+template <class Fixture>
+static void BM_PumpStreamClientToServer(benchmark::State& state) {
+  EchoTestService::AsyncService service;
+  std::unique_ptr<Fixture> fixture(new Fixture(&service));
+  {
+    EchoRequest send_request;
+    EchoRequest recv_request;
+    if (state.range(0) > 0) {
+      send_request.set_message(std::string(state.range(0), 'a'));
+    }
+    Status recv_status;
+    ServerContext svr_ctx;
+    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
+    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
+                              fixture->cq(), tag(0));
+    std::unique_ptr<EchoTestService::Stub> stub(
+        EchoTestService::NewStub(fixture->channel()));
+    ClientContext cli_ctx;
+    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
+    int need_tags = (1 << 0) | (1 << 1);
+    void* t;
+    bool ok;
+    while (need_tags) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      GPR_ASSERT(ok);
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+    response_rw.Read(&recv_request, tag(0));
+    while (state.KeepRunning()) {
+      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
+      request_rw->Write(send_request, tag(1));
+      while (true) {
+        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+        if (t == tag(0)) {
+          response_rw.Read(&recv_request, tag(0));
+        } else if (t == tag(1)) {
+          break;
+        } else {
+          GPR_ASSERT(false);
+        }
+      }
+    }
+    request_rw->WritesDone(tag(1));
+    need_tags = (1 << 0) | (1 << 1);
+    while (need_tags) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+  }
+  fixture->Finish(state);
+  fixture.reset();
+  state.SetBytesProcessed(state.range(0) * state.iterations());
+}
+
+template <class Fixture>
+static void BM_PumpStreamServerToClient(benchmark::State& state) {
+  EchoTestService::AsyncService service;
+  std::unique_ptr<Fixture> fixture(new Fixture(&service));
+  {
+    EchoResponse send_response;
+    EchoResponse recv_response;
+    if (state.range(0) > 0) {
+      send_response.set_message(std::string(state.range(0), 'a'));
+    }
+    Status recv_status;
+    ServerContext svr_ctx;
+    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
+    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
+                              fixture->cq(), tag(0));
+    std::unique_ptr<EchoTestService::Stub> stub(
+        EchoTestService::NewStub(fixture->channel()));
+    ClientContext cli_ctx;
+    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
+    int need_tags = (1 << 0) | (1 << 1);
+    void* t;
+    bool ok;
+    while (need_tags) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      GPR_ASSERT(ok);
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+    request_rw->Read(&recv_response, tag(0));
+    while (state.KeepRunning()) {
+      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
+      response_rw.Write(send_response, tag(1));
+      while (true) {
+        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+        if (t == tag(0)) {
+          request_rw->Read(&recv_response, tag(0));
+        } else if (t == tag(1)) {
+          break;
+        } else {
+          GPR_ASSERT(false);
+        }
+      }
+    }
+    response_rw.Finish(Status::OK, tag(1));
+    need_tags = (1 << 0) | (1 << 1);
+    while (need_tags) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+  }
+  fixture->Finish(state);
+  fixture.reset();
+  state.SetBytesProcessed(state.range(0) * state.iterations());
+}
+
+/*******************************************************************************
+ * CONFIGURATIONS
+ */
+
+BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2)
+    ->Range(0, 128 * 1024 * 1024);
+
+}  // namespace testing
+}  // namespace grpc
+
+BENCHMARK_MAIN();
diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc
new file mode 100644
index 0000000000..5011f06368
--- /dev/null
+++ b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc
@@ -0,0 +1,219 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* Benchmark gRPC end2end in various configurations */
+
+#include "src/core/lib/profiling/timers.h"
+#include "src/cpp/client/create_channel_internal.h"
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
+#include "test/cpp/microbenchmarks/fullstack_context_mutators.h"
+#include "test/cpp/microbenchmarks/fullstack_fixtures.h"
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+extern "C" {
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
+#include "test/core/util/trickle_endpoint.h"
+}
+
+namespace grpc {
+namespace testing {
+
+static void* tag(intptr_t x) { return reinterpret_cast<void*>(x); }
+
+class TrickledCHTTP2 : public EndpointPairFixture {
+ public:
+  TrickledCHTTP2(Service* service, size_t megabits_per_second)
+      : EndpointPairFixture(service, MakeEndpoints(megabits_per_second)) {}
+
+  void AddToLabel(std::ostream& out, benchmark::State& state) {
+    out << " writes/iter:"
+        << ((double)stats_.num_writes / (double)state.iterations())
+        << " cli_transport_stalls/iter:"
+        << ((double)
+                client_stats_.streams_stalled_due_to_transport_flow_control /
+            (double)state.iterations())
+        << " cli_stream_stalls/iter:"
+        << ((double)client_stats_.streams_stalled_due_to_stream_flow_control /
+            (double)state.iterations())
+        << " svr_transport_stalls/iter:"
+        << ((double)
+                server_stats_.streams_stalled_due_to_transport_flow_control /
+            (double)state.iterations())
+        << " svr_stream_stalls/iter:"
+        << ((double)server_stats_.streams_stalled_due_to_stream_flow_control /
+            (double)state.iterations());
+  }
+
+  void Step() {
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    size_t client_backlog =
+        grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.client);
+    size_t server_backlog =
+        grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.server);
+    grpc_exec_ctx_finish(&exec_ctx);
+
+    UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_,
+                client_backlog);
+    UpdateStats((grpc_chttp2_transport*)server_transport_, &server_stats_,
+                server_backlog);
+  }
+
+ private:
+  grpc_passthru_endpoint_stats stats_;
+  struct Stats {
+    int streams_stalled_due_to_stream_flow_control = 0;
+    int streams_stalled_due_to_transport_flow_control = 0;
+  };
+  Stats client_stats_;
+  Stats server_stats_;
+
+  grpc_endpoint_pair MakeEndpoints(size_t kilobits) {
+    grpc_endpoint_pair p;
+    grpc_passthru_endpoint_create(&p.client, &p.server, Library::get().rq(),
+                                  &stats_);
+    double bytes_per_second = 125.0 * kilobits;
+    p.client = grpc_trickle_endpoint_create(p.client, bytes_per_second);
+    p.server = grpc_trickle_endpoint_create(p.server, bytes_per_second);
+    return p;
+  }
+
+  void UpdateStats(grpc_chttp2_transport* t, Stats* s, size_t backlog) {
+    if (backlog == 0) {
+      if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_STREAM].head != NULL) {
+        s->streams_stalled_due_to_stream_flow_control++;
+      }
+      if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT].head != NULL) {
+        s->streams_stalled_due_to_transport_flow_control++;
+      }
+    }
+  }
+};
+
+// force library initialization
+auto& force_library_initialization = Library::get();
+
+static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok) {
+  while (true) {
+    switch (fixture->cq()->AsyncNext(
+        t, ok, gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
+                            gpr_time_from_micros(100, GPR_TIMESPAN)))) {
+      case CompletionQueue::TIMEOUT:
+        fixture->Step();
+        break;
+      case CompletionQueue::SHUTDOWN:
+        GPR_ASSERT(false);
+        break;
+      case CompletionQueue::GOT_EVENT:
+        return;
+    }
+  }
+}
+
+static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) {
+  EchoTestService::AsyncService service;
+  std::unique_ptr<TrickledCHTTP2> fixture(
+      new TrickledCHTTP2(&service, state.range(1)));
+  {
+    EchoResponse send_response;
+    EchoResponse recv_response;
+    if (state.range(0) > 0) {
+      send_response.set_message(std::string(state.range(0), 'a'));
+    }
+    Status recv_status;
+    ServerContext svr_ctx;
+    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
+    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
+                              fixture->cq(), tag(0));
+    std::unique_ptr<EchoTestService::Stub> stub(
+        EchoTestService::NewStub(fixture->channel()));
+    ClientContext cli_ctx;
+    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
+    int need_tags = (1 << 0) | (1 << 1);
+    void* t;
+    bool ok;
+    while (need_tags) {
+      TrickleCQNext(fixture.get(), &t, &ok);
+      GPR_ASSERT(ok);
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+    request_rw->Read(&recv_response, tag(0));
+    while (state.KeepRunning()) {
+      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
+      response_rw.Write(send_response, tag(1));
+      while (true) {
+        TrickleCQNext(fixture.get(), &t, &ok);
+        if (t == tag(0)) {
+          request_rw->Read(&recv_response, tag(0));
+        } else if (t == tag(1)) {
+          break;
+        } else {
+          GPR_ASSERT(false);
+        }
+      }
+    }
+    response_rw.Finish(Status::OK, tag(1));
+    need_tags = (1 << 0) | (1 << 1);
+    while (need_tags) {
+      TrickleCQNext(fixture.get(), &t, &ok);
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+  }
+  fixture->Finish(state);
+  fixture.reset();
+  state.SetBytesProcessed(state.range(0) * state.iterations());
+}
+
+/*******************************************************************************
+ * CONFIGURATIONS
+ */
+
+static void TrickleArgs(benchmark::internal::Benchmark* b) {
+  for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) {
+    for (int j = 1; j <= 128 * 1024 * 1024; j *= 8) {
+      double expected_time =
+          static_cast<double>(14 + i) / (125.0 * static_cast<double>(j));
+      if (expected_time > 0.01) continue;
+      b->Args({i, j});
+    }
+  }
+}
+
+BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs);
+}
+}
+
+BENCHMARK_MAIN();
diff --git a/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc
new file mode 100644
index 0000000000..e51d272b10
--- /dev/null
+++ b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc
@@ -0,0 +1,205 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* Benchmark gRPC end2end in various configurations */
+
+#include <sstream>
+
+#include "src/core/lib/profiling/timers.h"
+#include "src/cpp/client/create_channel_internal.h"
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
+#include "test/cpp/microbenchmarks/fullstack_context_mutators.h"
+#include "test/cpp/microbenchmarks/fullstack_fixtures.h"
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+
+namespace grpc {
+namespace testing {
+
+// force library initialization
+auto& force_library_initialization = Library::get();
+
+/*******************************************************************************
+ * BENCHMARKING KERNELS
+ */
+
+static void* tag(intptr_t x) { return reinterpret_cast<void*>(x); }
+
+template <class Fixture, class ClientContextMutator, class ServerContextMutator>
+static void BM_UnaryPingPong(benchmark::State& state) {
+  EchoTestService::AsyncService service;
+  std::unique_ptr<Fixture> fixture(new Fixture(&service));
+  EchoRequest send_request;
+  EchoResponse send_response;
+  EchoResponse recv_response;
+  if (state.range(0) > 0) {
+    send_request.set_message(std::string(state.range(0), 'a'));
+  }
+  if (state.range(1) > 0) {
+    send_response.set_message(std::string(state.range(1), 'a'));
+  }
+  Status recv_status;
+  struct ServerEnv {
+    ServerContext ctx;
+    EchoRequest recv_request;
+    grpc::ServerAsyncResponseWriter<EchoResponse> response_writer;
+    ServerEnv() : response_writer(&ctx) {}
+  };
+  uint8_t server_env_buffer[2 * sizeof(ServerEnv)];
+  ServerEnv* server_env[2] = {
+      reinterpret_cast<ServerEnv*>(server_env_buffer),
+      reinterpret_cast<ServerEnv*>(server_env_buffer + sizeof(ServerEnv))};
+  new (server_env[0]) ServerEnv;
+  new (server_env[1]) ServerEnv;
+  service.RequestEcho(&server_env[0]->ctx, &server_env[0]->recv_request,
+                      &server_env[0]->response_writer, fixture->cq(),
+                      fixture->cq(), tag(0));
+  service.RequestEcho(&server_env[1]->ctx, &server_env[1]->recv_request,
+                      &server_env[1]->response_writer, fixture->cq(),
+                      fixture->cq(), tag(1));
+  std::unique_ptr<EchoTestService::Stub> stub(
+      EchoTestService::NewStub(fixture->channel()));
+  while (state.KeepRunning()) {
+    GPR_TIMER_SCOPE("BenchmarkCycle", 0);
+    recv_response.Clear();
+    ClientContext cli_ctx;
+    ClientContextMutator cli_ctx_mut(&cli_ctx);
+    std::unique_ptr<ClientAsyncResponseReader<EchoResponse>> response_reader(
+        stub->AsyncEcho(&cli_ctx, send_request, fixture->cq()));
+    void* t;
+    bool ok;
+    GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+    GPR_ASSERT(ok);
+    GPR_ASSERT(t == tag(0) || t == tag(1));
+    intptr_t slot = reinterpret_cast<intptr_t>(t);
+    ServerEnv* senv = server_env[slot];
+    ServerContextMutator svr_ctx_mut(&senv->ctx);
+    senv->response_writer.Finish(send_response, Status::OK, tag(3));
+    response_reader->Finish(&recv_response, &recv_status, tag(4));
+    for (int i = (1 << 3) | (1 << 4); i != 0;) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      GPR_ASSERT(ok);
+      int tagnum = (int)reinterpret_cast<intptr_t>(t);
+      GPR_ASSERT(i & (1 << tagnum));
+      i -= 1 << tagnum;
+    }
+    GPR_ASSERT(recv_status.ok());
+
+    senv->~ServerEnv();
+    senv = new (senv) ServerEnv();
+    service.RequestEcho(&senv->ctx, &senv->recv_request, &senv->response_writer,
+                        fixture->cq(), fixture->cq(), tag(slot));
+  }
+  fixture->Finish(state);
+  fixture.reset();
+  server_env[0]->~ServerEnv();
+  server_env[1]->~ServerEnv();
+  state.SetBytesProcessed(state.range(0) * state.iterations() +
+                          state.range(1) * state.iterations());
+}
+
+/*******************************************************************************
+ * CONFIGURATIONS
+ */
+
+static void SweepSizesArgs(benchmark::internal::Benchmark* b) {
+  b->Args({0, 0});
+  for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) {
+    b->Args({i, 0});
+    b->Args({0, i});
+    b->Args({i, i});
+  }
+}
+
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, TCP, NoOpMutator, NoOpMutator)
+    ->Apply(SweepSizesArgs);
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, UDS, NoOpMutator, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, SockPair, NoOpMutator, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, NoOpMutator)
+    ->Apply(SweepSizesArgs);
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomBinaryMetadata<10>, 1>, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomBinaryMetadata<31>, 1>, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomBinaryMetadata<100>, 1>,
+                   NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomBinaryMetadata<10>, 2>, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomBinaryMetadata<31>, 2>, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomBinaryMetadata<100>, 2>,
+                   NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
+                   Server_AddInitialMetadata<RandomBinaryMetadata<10>, 1>)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
+                   Server_AddInitialMetadata<RandomBinaryMetadata<31>, 1>)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
+                   Server_AddInitialMetadata<RandomBinaryMetadata<100>, 1>)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomAsciiMetadata<10>, 1>, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomAsciiMetadata<31>, 1>, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomAsciiMetadata<100>, 1>, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
+                   Server_AddInitialMetadata<RandomAsciiMetadata<10>, 1>)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
+                   Server_AddInitialMetadata<RandomAsciiMetadata<31>, 1>)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
+                   Server_AddInitialMetadata<RandomAsciiMetadata<100>, 1>)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
+                   Server_AddInitialMetadata<RandomAsciiMetadata<10>, 100>)
+    ->Args({0, 0});
+
+}  // namespace testing
+}  // namespace grpc
+
+BENCHMARK_MAIN();
diff --git a/test/cpp/microbenchmarks/bm_metadata.cc b/test/cpp/microbenchmarks/bm_metadata.cc
index 7f81fbabcc..34874b57f5 100644
--- a/test/cpp/microbenchmarks/bm_metadata.cc
+++ b/test/cpp/microbenchmarks/bm_metadata.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2017, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -36,65 +36,90 @@
 #include <grpc/grpc.h>
 
 extern "C" {
+#include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/transport/metadata.h"
 #include "src/core/lib/transport/static_metadata.h"
+#include "src/core/lib/transport/transport.h"
 }
 
+#include "test/cpp/microbenchmarks/helpers.h"
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
-static class InitializeStuff {
- public:
-  InitializeStuff() { grpc_init(); }
-  ~InitializeStuff() { grpc_shutdown(); }
-} initialize_stuff;
+auto& force_library_initialization = Library::get();
 
 static void BM_SliceFromStatic(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     benchmark::DoNotOptimize(grpc_slice_from_static_string("abc"));
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_SliceFromStatic);
 
 static void BM_SliceFromCopied(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     grpc_slice_unref(grpc_slice_from_copied_string("abc"));
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_SliceFromCopied);
 
+static void BM_SliceFromStreamOwnedBuffer(benchmark::State& state) {
+  grpc_stream_refcount r;
+  GRPC_STREAM_REF_INIT(&r, 1, NULL, NULL, "test");
+  char buffer[64];
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  while (state.KeepRunning()) {
+    grpc_slice_unref_internal(&exec_ctx, grpc_slice_from_stream_owned_buffer(
+                                             &r, buffer, sizeof(buffer)));
+  }
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+BENCHMARK(BM_SliceFromStreamOwnedBuffer);
+
 static void BM_SliceIntern(benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice slice = grpc_slice_from_static_string("abc");
   while (state.KeepRunning()) {
     grpc_slice_unref(grpc_slice_intern(slice));
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_SliceIntern);
 
 static void BM_SliceReIntern(benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice slice = grpc_slice_intern(grpc_slice_from_static_string("abc"));
   while (state.KeepRunning()) {
     grpc_slice_unref(grpc_slice_intern(slice));
   }
   grpc_slice_unref(slice);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_SliceReIntern);
 
 static void BM_SliceInternStaticMetadata(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     grpc_slice_intern(GRPC_MDSTR_GZIP);
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_SliceInternStaticMetadata);
 
 static void BM_SliceInternEqualToStaticMetadata(benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice slice = grpc_slice_from_static_string("gzip");
   while (state.KeepRunning()) {
     grpc_slice_intern(slice);
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_SliceInternEqualToStaticMetadata);
 
 static void BM_MetadataFromNonInternedSlices(benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = grpc_slice_from_static_string("key");
   gpr_slice v = grpc_slice_from_static_string("value");
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -102,10 +127,12 @@ static void BM_MetadataFromNonInternedSlices(benchmark::State& state) {
     GRPC_MDELEM_UNREF(&exec_ctx, grpc_mdelem_create(&exec_ctx, k, v, NULL));
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromNonInternedSlices);
 
 static void BM_MetadataFromInternedSlices(benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key"));
   gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value"));
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -115,11 +142,13 @@ static void BM_MetadataFromInternedSlices(benchmark::State& state) {
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_slice_unref(k);
   grpc_slice_unref(v);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromInternedSlices);
 
 static void BM_MetadataFromInternedSlicesAlreadyInIndex(
     benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key"));
   gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value"));
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -131,10 +160,12 @@ static void BM_MetadataFromInternedSlicesAlreadyInIndex(
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_slice_unref(k);
   grpc_slice_unref(v);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromInternedSlicesAlreadyInIndex);
 
 static void BM_MetadataFromInternedKey(benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key"));
   gpr_slice v = grpc_slice_from_static_string("value");
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -143,11 +174,13 @@ static void BM_MetadataFromInternedKey(benchmark::State& state) {
   }
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_slice_unref(k);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromInternedKey);
 
 static void BM_MetadataFromNonInternedSlicesWithBackingStore(
     benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = grpc_slice_from_static_string("key");
   gpr_slice v = grpc_slice_from_static_string("value");
   char backing_store[sizeof(grpc_mdelem_data)];
@@ -159,11 +192,13 @@ static void BM_MetadataFromNonInternedSlicesWithBackingStore(
                            reinterpret_cast<grpc_mdelem_data*>(backing_store)));
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromNonInternedSlicesWithBackingStore);
 
 static void BM_MetadataFromInternedSlicesWithBackingStore(
     benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key"));
   gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value"));
   char backing_store[sizeof(grpc_mdelem_data)];
@@ -177,11 +212,13 @@ static void BM_MetadataFromInternedSlicesWithBackingStore(
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_slice_unref(k);
   grpc_slice_unref(v);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromInternedSlicesWithBackingStore);
 
 static void BM_MetadataFromInternedKeyWithBackingStore(
     benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key"));
   gpr_slice v = grpc_slice_from_static_string("value");
   char backing_store[sizeof(grpc_mdelem_data)];
@@ -194,10 +231,12 @@ static void BM_MetadataFromInternedKeyWithBackingStore(
   }
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_slice_unref(k);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromInternedKeyWithBackingStore);
 
 static void BM_MetadataFromStaticMetadataStrings(benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = GRPC_MDSTR_STATUS;
   gpr_slice v = GRPC_MDSTR_200;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -206,11 +245,13 @@ static void BM_MetadataFromStaticMetadataStrings(benchmark::State& state) {
   }
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_slice_unref(k);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromStaticMetadataStrings);
 
 static void BM_MetadataFromStaticMetadataStringsNotIndexed(
     benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = GRPC_MDSTR_STATUS;
   gpr_slice v = GRPC_MDSTR_GZIP;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -219,10 +260,12 @@ static void BM_MetadataFromStaticMetadataStringsNotIndexed(
   }
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_slice_unref(k);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromStaticMetadataStringsNotIndexed);
 
 static void BM_MetadataRefUnrefExternal(benchmark::State& state) {
+  TrackCounters track_counters;
   char backing_store[sizeof(grpc_mdelem_data)];
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_mdelem el =
@@ -234,10 +277,12 @@ static void BM_MetadataRefUnrefExternal(benchmark::State& state) {
   }
   GRPC_MDELEM_UNREF(&exec_ctx, el);
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataRefUnrefExternal);
 
 static void BM_MetadataRefUnrefInterned(benchmark::State& state) {
+  TrackCounters track_counters;
   char backing_store[sizeof(grpc_mdelem_data)];
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key"));
@@ -251,10 +296,12 @@ static void BM_MetadataRefUnrefInterned(benchmark::State& state) {
   }
   GRPC_MDELEM_UNREF(&exec_ctx, el);
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataRefUnrefInterned);
 
 static void BM_MetadataRefUnrefAllocated(benchmark::State& state) {
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_mdelem el =
       grpc_mdelem_create(&exec_ctx, grpc_slice_from_static_string("a"),
@@ -264,10 +311,12 @@ static void BM_MetadataRefUnrefAllocated(benchmark::State& state) {
   }
   GRPC_MDELEM_UNREF(&exec_ctx, el);
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataRefUnrefAllocated);
 
 static void BM_MetadataRefUnrefStatic(benchmark::State& state) {
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_mdelem el =
       grpc_mdelem_create(&exec_ctx, GRPC_MDSTR_STATUS, GRPC_MDSTR_200, NULL);
@@ -276,6 +325,7 @@ static void BM_MetadataRefUnrefStatic(benchmark::State& state) {
   }
   GRPC_MDELEM_UNREF(&exec_ctx, el);
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataRefUnrefStatic);
 
diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc
new file mode 100644
index 0000000000..0f3d3cef66
--- /dev/null
+++ b/test/cpp/microbenchmarks/bm_pollset.cc
@@ -0,0 +1,254 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* Test out pollset latencies */
+
+#include <grpc/grpc.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/useful.h>
+
+extern "C" {
+#include "src/core/lib/iomgr/ev_posix.h"
+#include "src/core/lib/iomgr/pollset.h"
+#include "src/core/lib/iomgr/port.h"
+#include "src/core/lib/iomgr/wakeup_fd_posix.h"
+}
+
+#include "test/cpp/microbenchmarks/helpers.h"
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+
+#include <string.h>
+
+#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <unistd.h>
+#endif
+
+auto& force_library_initialization = Library::get();
+
+static void shutdown_ps(grpc_exec_ctx* exec_ctx, void* ps, grpc_error* error) {
+  grpc_pollset_destroy(static_cast<grpc_pollset*>(ps));
+}
+
+static void BM_CreateDestroyPollset(benchmark::State& state) {
+  TrackCounters track_counters;
+  size_t ps_sz = grpc_pollset_size();
+  grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_malloc(ps_sz));
+  gpr_mu* mu;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_closure shutdown_ps_closure;
+  grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps,
+                    grpc_schedule_on_exec_ctx);
+  while (state.KeepRunning()) {
+    memset(ps, 0, ps_sz);
+    grpc_pollset_init(ps, &mu);
+    gpr_mu_lock(mu);
+    grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure);
+    gpr_mu_unlock(mu);
+    grpc_exec_ctx_flush(&exec_ctx);
+  }
+  grpc_exec_ctx_finish(&exec_ctx);
+  gpr_free(ps);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_CreateDestroyPollset);
+
+#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL
+static void BM_PollEmptyPollset_SpeedOfLight(benchmark::State& state) {
+  // equivalent to BM_PollEmptyPollset, but just use the OS primitives to guage
+  // what the speed of light would be if we abstracted perfectly
+  TrackCounters track_counters;
+  int epfd = epoll_create1(0);
+  GPR_ASSERT(epfd != -1);
+  size_t nev = state.range(0);
+  size_t nfd = state.range(1);
+  epoll_event* ev = new epoll_event[nev];
+  std::vector<int> fds;
+  for (size_t i = 0; i < nfd; i++) {
+    fds.push_back(eventfd(0, 0));
+    epoll_event ev;
+    ev.events = EPOLLIN;
+    epoll_ctl(epfd, EPOLL_CTL_ADD, fds.back(), &ev);
+  }
+  while (state.KeepRunning()) {
+    epoll_wait(epfd, ev, nev, 0);
+  }
+  for (auto fd : fds) {
+    close(fd);
+  }
+  close(epfd);
+  delete[] ev;
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_PollEmptyPollset_SpeedOfLight)
+    ->Args({1, 0})
+    ->Args({1, 1})
+    ->Args({1, 10})
+    ->Args({1, 100})
+    ->Args({1, 1000})
+    ->Args({1, 10000})
+    ->Args({1, 100000})
+    ->Args({10, 1})
+    ->Args({100, 1})
+    ->Args({1000, 1});
+#endif
+
+static void BM_PollEmptyPollset(benchmark::State& state) {
+  TrackCounters track_counters;
+  size_t ps_sz = grpc_pollset_size();
+  grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_zalloc(ps_sz));
+  gpr_mu* mu;
+  grpc_pollset_init(ps, &mu);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  gpr_timespec now = gpr_time_0(GPR_CLOCK_MONOTONIC);
+  gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC);
+  gpr_mu_lock(mu);
+  while (state.KeepRunning()) {
+    grpc_pollset_worker* worker;
+    GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, &worker, now, deadline));
+  }
+  grpc_closure shutdown_ps_closure;
+  grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps,
+                    grpc_schedule_on_exec_ctx);
+  grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure);
+  gpr_mu_unlock(mu);
+  grpc_exec_ctx_finish(&exec_ctx);
+  gpr_free(ps);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_PollEmptyPollset);
+
+class Closure : public grpc_closure {
+ public:
+  virtual ~Closure() {}
+};
+
+template <class F>
+Closure* MakeClosure(F f, grpc_closure_scheduler* scheduler) {
+  struct C : public Closure {
+    C(F f, grpc_closure_scheduler* scheduler) : f_(f) {
+      grpc_closure_init(this, C::cbfn, this, scheduler);
+    }
+    static void cbfn(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {
+      C* p = static_cast<C*>(arg);
+      p->f_();
+    }
+    F f_;
+  };
+  return new C(f, scheduler);
+}
+
+#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL
+static void BM_SingleThreadPollOneFd_SpeedOfLight(benchmark::State& state) {
+  // equivalent to BM_PollEmptyPollset, but just use the OS primitives to guage
+  // what the speed of light would be if we abstracted perfectly
+  TrackCounters track_counters;
+  int epfd = epoll_create1(0);
+  GPR_ASSERT(epfd != -1);
+  epoll_event ev[100];
+  int fd = eventfd(0, EFD_NONBLOCK);
+  ev[0].events = EPOLLIN;
+  epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev[0]);
+  while (state.KeepRunning()) {
+    int err;
+    do {
+      err = eventfd_write(fd, 1);
+    } while (err < 0 && errno == EINTR);
+    GPR_ASSERT(err == 0);
+    do {
+      err = epoll_wait(epfd, ev, GPR_ARRAY_SIZE(ev), 0);
+    } while (err < 0 && errno == EINTR);
+    GPR_ASSERT(err == 1);
+    eventfd_t value;
+    do {
+      err = eventfd_read(fd, &value);
+    } while (err < 0 && errno == EINTR);
+    GPR_ASSERT(err == 0);
+  }
+  close(fd);
+  close(epfd);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_SingleThreadPollOneFd_SpeedOfLight);
+#endif
+
+static void BM_SingleThreadPollOneFd(benchmark::State& state) {
+  TrackCounters track_counters;
+  size_t ps_sz = grpc_pollset_size();
+  grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_zalloc(ps_sz));
+  gpr_mu* mu;
+  grpc_pollset_init(ps, &mu);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  gpr_timespec now = gpr_time_0(GPR_CLOCK_MONOTONIC);
+  gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
+  grpc_wakeup_fd wakeup_fd;
+  GRPC_ERROR_UNREF(grpc_wakeup_fd_init(&wakeup_fd));
+  grpc_fd* wakeup = grpc_fd_create(wakeup_fd.read_fd, "wakeup_read");
+  grpc_pollset_add_fd(&exec_ctx, ps, wakeup);
+  bool done = false;
+  Closure* continue_closure = MakeClosure(
+      [&]() {
+        GRPC_ERROR_UNREF(grpc_wakeup_fd_consume_wakeup(&wakeup_fd));
+        if (!state.KeepRunning()) {
+          done = true;
+          return;
+        }
+        GRPC_ERROR_UNREF(grpc_wakeup_fd_wakeup(&wakeup_fd));
+        grpc_fd_notify_on_read(&exec_ctx, wakeup, continue_closure);
+      },
+      grpc_schedule_on_exec_ctx);
+  GRPC_ERROR_UNREF(grpc_wakeup_fd_wakeup(&wakeup_fd));
+  grpc_fd_notify_on_read(&exec_ctx, wakeup, continue_closure);
+  gpr_mu_lock(mu);
+  while (!done) {
+    grpc_pollset_worker* worker;
+    GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, &worker, now, deadline));
+  }
+  grpc_fd_orphan(&exec_ctx, wakeup, NULL, NULL, "done");
+  wakeup_fd.read_fd = 0;
+  grpc_closure shutdown_ps_closure;
+  grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps,
+                    grpc_schedule_on_exec_ctx);
+  grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure);
+  gpr_mu_unlock(mu);
+  grpc_exec_ctx_finish(&exec_ctx);
+  grpc_wakeup_fd_destroy(&wakeup_fd);
+  gpr_free(ps);
+  track_counters.Finish(state);
+  delete continue_closure;
+}
+BENCHMARK(BM_SingleThreadPollOneFd);
+
+BENCHMARK_MAIN();
diff --git a/test/cpp/microbenchmarks/fullstack_context_mutators.h b/test/cpp/microbenchmarks/fullstack_context_mutators.h
new file mode 100644
index 0000000000..676f9aa1cc
--- /dev/null
+++ b/test/cpp/microbenchmarks/fullstack_context_mutators.h
@@ -0,0 +1,158 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef TEST_CPP_MICROBENCHMARKS_FULLSTACK_CONTEXT_MUTATORS_H
+#define TEST_CPP_MICROBENCHMARKS_FULLSTACK_CONTEXT_MUTATORS_H
+
+#include <grpc++/channel.h>
+#include <grpc++/create_channel.h>
+#include <grpc++/security/credentials.h>
+#include <grpc++/security/server_credentials.h>
+#include <grpc++/server.h>
+#include <grpc++/server_builder.h>
+#include <grpc++/server_context.h>
+#include <grpc/support/log.h>
+
+#include "test/cpp/microbenchmarks/helpers.h"
+
+namespace grpc {
+namespace testing {
+
+/*******************************************************************************
+ * CONTEXT MUTATORS
+ */
+
+static const int kPregenerateKeyCount = 100000;
+
+template <class F>
+auto MakeVector(size_t length, F f) -> std::vector<decltype(f())> {
+  std::vector<decltype(f())> out;
+  out.reserve(length);
+  for (size_t i = 0; i < length; i++) {
+    out.push_back(f());
+  }
+  return out;
+}
+
+class NoOpMutator {
+ public:
+  template <class ContextType>
+  NoOpMutator(ContextType* context) {}
+};
+
+template <int length>
+class RandomBinaryMetadata {
+ public:
+  static const grpc::string& Key() { return kKey; }
+
+  static const grpc::string& Value() {
+    return kValues[rand() % kValues.size()];
+  }
+
+ private:
+  static const grpc::string kKey;
+  static const std::vector<grpc::string> kValues;
+
+  static grpc::string GenerateOneString() {
+    grpc::string s;
+    s.reserve(length + 1);
+    for (int i = 0; i < length; i++) {
+      s += (char)rand();
+    }
+    return s;
+  }
+};
+
+template <int length>
+class RandomAsciiMetadata {
+ public:
+  static const grpc::string& Key() { return kKey; }
+
+  static const grpc::string& Value() {
+    return kValues[rand() % kValues.size()];
+  }
+
+ private:
+  static const grpc::string kKey;
+  static const std::vector<grpc::string> kValues;
+
+  static grpc::string GenerateOneString() {
+    grpc::string s;
+    s.reserve(length + 1);
+    for (int i = 0; i < length; i++) {
+      s += (char)(rand() % 26 + 'a');
+    }
+    return s;
+  }
+};
+
+template <class Generator, int kNumKeys>
+class Client_AddMetadata : public NoOpMutator {
+ public:
+  Client_AddMetadata(ClientContext* context) : NoOpMutator(context) {
+    for (int i = 0; i < kNumKeys; i++) {
+      context->AddMetadata(Generator::Key(), Generator::Value());
+    }
+  }
+};
+
+template <class Generator, int kNumKeys>
+class Server_AddInitialMetadata : public NoOpMutator {
+ public:
+  Server_AddInitialMetadata(ServerContext* context) : NoOpMutator(context) {
+    for (int i = 0; i < kNumKeys; i++) {
+      context->AddInitialMetadata(Generator::Key(), Generator::Value());
+    }
+  }
+};
+
+// static initialization
+
+template <int length>
+const grpc::string RandomBinaryMetadata<length>::kKey = "foo-bin";
+
+template <int length>
+const std::vector<grpc::string> RandomBinaryMetadata<length>::kValues =
+    MakeVector(kPregenerateKeyCount, GenerateOneString);
+
+template <int length>
+const grpc::string RandomAsciiMetadata<length>::kKey = "foo";
+
+template <int length>
+const std::vector<grpc::string> RandomAsciiMetadata<length>::kValues =
+    MakeVector(kPregenerateKeyCount, GenerateOneString);
+
+}  // namespace testing
+}  // namespace grpc
+
+#endif
diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h
new file mode 100644
index 0000000000..dc29701059
--- /dev/null
+++ b/test/cpp/microbenchmarks/fullstack_fixtures.h
@@ -0,0 +1,244 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef TEST_CPP_MICROBENCHMARKS_FULLSTACK_FIXTURES_H
+#define TEST_CPP_MICROBENCHMARKS_FULLSTACK_FIXTURES_H
+
+#include <grpc++/channel.h>
+#include <grpc++/create_channel.h>
+#include <grpc++/security/credentials.h>
+#include <grpc++/security/server_credentials.h>
+#include <grpc++/server.h>
+#include <grpc++/server_builder.h>
+#include <grpc/support/log.h>
+
+extern "C" {
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/endpoint_pair.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/tcp_posix.h"
+#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/surface/completion_queue.h"
+#include "src/core/lib/surface/server.h"
+#include "test/core/util/passthru_endpoint.h"
+#include "test/core/util/port.h"
+}
+
+#include "test/cpp/microbenchmarks/helpers.h"
+
+namespace grpc {
+namespace testing {
+
+static void ApplyCommonServerBuilderConfig(ServerBuilder* b) {
+  b->SetMaxReceiveMessageSize(INT_MAX);
+  b->SetMaxSendMessageSize(INT_MAX);
+}
+
+static void ApplyCommonChannelArguments(ChannelArguments* c) {
+  c->SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, INT_MAX);
+  c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX);
+}
+
+class BaseFixture : public TrackCounters {};
+
+class FullstackFixture : public BaseFixture {
+ public:
+  FullstackFixture(Service* service, const grpc::string& address) {
+    ServerBuilder b;
+    b.AddListeningPort(address, InsecureServerCredentials());
+    cq_ = b.AddCompletionQueue(true);
+    b.RegisterService(service);
+    ApplyCommonServerBuilderConfig(&b);
+    server_ = b.BuildAndStart();
+    ChannelArguments args;
+    ApplyCommonChannelArguments(&args);
+    channel_ = CreateCustomChannel(address, InsecureChannelCredentials(), args);
+  }
+
+  virtual ~FullstackFixture() {
+    server_->Shutdown();
+    cq_->Shutdown();
+    void* tag;
+    bool ok;
+    while (cq_->Next(&tag, &ok)) {
+    }
+  }
+
+  ServerCompletionQueue* cq() { return cq_.get(); }
+  std::shared_ptr<Channel> channel() { return channel_; }
+
+ private:
+  std::unique_ptr<Server> server_;
+  std::unique_ptr<ServerCompletionQueue> cq_;
+  std::shared_ptr<Channel> channel_;
+};
+
+class TCP : public FullstackFixture {
+ public:
+  TCP(Service* service) : FullstackFixture(service, MakeAddress()) {}
+
+ private:
+  static grpc::string MakeAddress() {
+    int port = grpc_pick_unused_port_or_die();
+    std::stringstream addr;
+    addr << "localhost:" << port;
+    return addr.str();
+  }
+};
+
+class UDS : public FullstackFixture {
+ public:
+  UDS(Service* service) : FullstackFixture(service, MakeAddress()) {}
+
+ private:
+  static grpc::string MakeAddress() {
+    int port = grpc_pick_unused_port_or_die();  // just for a unique id - not a
+                                                // real port
+    std::stringstream addr;
+    addr << "unix:/tmp/bm_fullstack." << port;
+    return addr.str();
+  }
+};
+
+class EndpointPairFixture : public BaseFixture {
+ public:
+  EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints)
+      : endpoint_pair_(endpoints) {
+    ServerBuilder b;
+    cq_ = b.AddCompletionQueue(true);
+    b.RegisterService(service);
+    ApplyCommonServerBuilderConfig(&b);
+    server_ = b.BuildAndStart();
+
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+
+    /* add server endpoint to server_
+     * */
+    {
+      const grpc_channel_args* server_args =
+          grpc_server_get_channel_args(server_->c_server());
+      server_transport_ = grpc_create_chttp2_transport(
+          &exec_ctx, server_args, endpoints.server, 0 /* is_client */);
+
+      grpc_pollset** pollsets;
+      size_t num_pollsets = 0;
+      grpc_server_get_pollsets(server_->c_server(), &pollsets, &num_pollsets);
+
+      for (size_t i = 0; i < num_pollsets; i++) {
+        grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]);
+      }
+
+      grpc_server_setup_transport(&exec_ctx, server_->c_server(),
+                                  server_transport_, NULL, server_args);
+      grpc_chttp2_transport_start_reading(&exec_ctx, server_transport_, NULL);
+    }
+
+    /* create channel */
+    {
+      ChannelArguments args;
+      args.SetString(GRPC_ARG_DEFAULT_AUTHORITY, "test.authority");
+      ApplyCommonChannelArguments(&args);
+
+      grpc_channel_args c_args = args.c_channel_args();
+      client_transport_ =
+          grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1);
+      GPR_ASSERT(client_transport_);
+      grpc_channel* channel =
+          grpc_channel_create(&exec_ctx, "target", &c_args,
+                              GRPC_CLIENT_DIRECT_CHANNEL, client_transport_);
+      grpc_chttp2_transport_start_reading(&exec_ctx, client_transport_, NULL);
+
+      channel_ = CreateChannelInternal("", channel);
+    }
+
+    grpc_exec_ctx_finish(&exec_ctx);
+  }
+
+  virtual ~EndpointPairFixture() {
+    server_->Shutdown();
+    cq_->Shutdown();
+    void* tag;
+    bool ok;
+    while (cq_->Next(&tag, &ok)) {
+    }
+  }
+
+  ServerCompletionQueue* cq() { return cq_.get(); }
+  std::shared_ptr<Channel> channel() { return channel_; }
+
+ protected:
+  grpc_endpoint_pair endpoint_pair_;
+  grpc_transport* client_transport_;
+  grpc_transport* server_transport_;
+
+ private:
+  std::unique_ptr<Server> server_;
+  std::unique_ptr<ServerCompletionQueue> cq_;
+  std::shared_ptr<Channel> channel_;
+};
+
+class SockPair : public EndpointPairFixture {
+ public:
+  SockPair(Service* service)
+      : EndpointPairFixture(service, grpc_iomgr_create_endpoint_pair(
+                                         "test", Library::get().rq(), 8192)) {}
+};
+
+class InProcessCHTTP2 : public EndpointPairFixture {
+ public:
+  InProcessCHTTP2(Service* service)
+      : EndpointPairFixture(service, MakeEndpoints()) {}
+
+  void AddToLabel(std::ostream& out, benchmark::State& state) {
+    EndpointPairFixture::AddToLabel(out, state);
+    out << " writes/iter:"
+        << ((double)stats_.num_writes / (double)state.iterations());
+  }
+
+ private:
+  grpc_passthru_endpoint_stats stats_;
+
+  grpc_endpoint_pair MakeEndpoints() {
+    grpc_endpoint_pair p;
+    grpc_passthru_endpoint_create(&p.client, &p.server, Library::get().rq(),
+                                  &stats_);
+    return p;
+  }
+};
+
+}  // namespace testing
+}  // namespace grpc
+
+#endif
diff --git a/test/cpp/microbenchmarks/helpers.cc b/test/cpp/microbenchmarks/helpers.cc
new file mode 100644
index 0000000000..d277c5984c
--- /dev/null
+++ b/test/cpp/microbenchmarks/helpers.cc
@@ -0,0 +1,65 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "test/cpp/microbenchmarks/helpers.h"
+
+void TrackCounters::Finish(benchmark::State &state) {
+  std::ostringstream out;
+  AddToLabel(out, state);
+  auto label = out.str();
+  if (label.length() && label[0] == ' ') {
+    label = label.substr(1);
+  }
+  state.SetLabel(label);
+}
+
+void TrackCounters::AddToLabel(std::ostream &out, benchmark::State &state) {
+#ifdef GPR_LOW_LEVEL_COUNTERS
+  grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot();
+  out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) -
+                                     mu_locks_at_start_) /
+                            (double)state.iterations())
+      << " atm_cas/iter:"
+      << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) -
+                   atm_cas_at_start_) /
+          (double)state.iterations())
+      << " atm_add/iter:"
+      << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) -
+                   atm_add_at_start_) /
+          (double)state.iterations())
+      << " allocs/iter:"
+      << ((double)(counters_at_end.total_allocs_absolute -
+                   counters_at_start_.total_allocs_absolute) /
+          (double)state.iterations());
+#endif
+}
diff --git a/test/cpp/microbenchmarks/helpers.h b/test/cpp/microbenchmarks/helpers.h
new file mode 100644
index 0000000000..f44b7cf83a
--- /dev/null
+++ b/test/cpp/microbenchmarks/helpers.h
@@ -0,0 +1,93 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef TEST_CPP_MICROBENCHMARKS_COUNTERS_H
+#define TEST_CPP_MICROBENCHMARKS_COUNTERS_H
+
+#include <sstream>
+
+extern "C" {
+#include <grpc/support/port_platform.h>
+#include "test/core/util/memory_counters.h"
+}
+
+#include <grpc++/impl/grpc_library.h>
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+
+class Library {
+ public:
+  static Library& get() {
+    static Library lib;
+    return lib;
+  }
+
+  grpc_resource_quota* rq() { return rq_; }
+
+ private:
+  Library() {
+#ifdef GPR_LOW_LEVEL_COUNTERS
+    grpc_memory_counters_init();
+#endif
+    init_lib_.init();
+    rq_ = grpc_resource_quota_create("bm");
+  }
+
+  ~Library() { init_lib_.shutdown(); }
+
+  grpc::internal::GrpcLibrary init_lib_;
+  grpc_resource_quota* rq_;
+};
+
+#ifdef GPR_LOW_LEVEL_COUNTERS
+extern "C" gpr_atm gpr_mu_locks;
+extern "C" gpr_atm gpr_counter_atm_cas;
+extern "C" gpr_atm gpr_counter_atm_add;
+#endif
+
+class TrackCounters {
+ public:
+  virtual void Finish(benchmark::State& state);
+  virtual void AddToLabel(std::ostream& out, benchmark::State& state);
+
+ private:
+#ifdef GPR_LOW_LEVEL_COUNTERS
+  const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks);
+  const size_t atm_cas_at_start_ =
+      gpr_atm_no_barrier_load(&gpr_counter_atm_cas);
+  const size_t atm_add_at_start_ =
+      gpr_atm_no_barrier_load(&gpr_counter_atm_add);
+  grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot();
+#endif
+};
+
+#endif