Merge pull request #10070 from ctiller/fmac

Redo bm_fullstack split, fixing mac build

15 files changed, 1521 insertions, 1168 deletions

diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc
index e2e5bbbe00..f25bcd2df8 100644
--- a/test/cpp/microbenchmarks/bm_call_create.cc
+++ b/test/cpp/microbenchmarks/bm_call_create.cc
@@ -55,12 +55,10 @@ extern "C" {
 #include "src/core/lib/transport/transport_impl.h"
 }
 
+#include "test/cpp/microbenchmarks/helpers.h"
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
-static struct Init {
-  Init() { grpc_init(); }
-  ~Init() { grpc_shutdown(); }
-} g_init;
+auto &force_library_initialization = Library::get();
 
 class BaseChannelFixture {
  public:
@@ -89,6 +87,7 @@ class LameChannel : public BaseChannelFixture {
 
 template <class Fixture>
 static void BM_CallCreateDestroy(benchmark::State &state) {
+  TrackCounters track_counters;
   Fixture fixture;
   grpc_completion_queue *cq = grpc_completion_queue_create(NULL);
   gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
@@ -100,6 +99,7 @@ static void BM_CallCreateDestroy(benchmark::State &state) {
         deadline, NULL));
   }
   grpc_completion_queue_destroy(cq);
+  track_counters.Finish(state);
 }
 
 BENCHMARK_TEMPLATE(BM_CallCreateDestroy, InsecureChannel);
@@ -316,6 +316,7 @@ class SendEmptyMetadata {
 // perform on said filter.
 template <class Fixture, class TestOp>
 static void BM_IsolatedFilter(benchmark::State &state) {
+  TrackCounters track_counters;
   Fixture fixture;
   std::ostringstream label;
 
@@ -371,6 +372,7 @@ static void BM_IsolatedFilter(benchmark::State &state) {
   gpr_free(call_stack);
 
   state.SetLabel(label.str());
+  track_counters.Finish(state);
 }
 
 typedef Fixture<nullptr, 0> NoFilter;
diff --git a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc
index 5fb3f37130..563db758f7 100644
--- a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc
+++ b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc
@@ -42,18 +42,17 @@ extern "C" {
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/transport/static_metadata.h"
 }
+#include "test/cpp/microbenchmarks/helpers.h"
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
-static struct Init {
-  Init() { grpc_init(); }
-  ~Init() { grpc_shutdown(); }
-} g_init;
+auto &force_library_initialization = Library::get();
 
 ////////////////////////////////////////////////////////////////////////////////
 // HPACK encoder
 //
 
 static void BM_HpackEncoderInitDestroy(benchmark::State &state) {
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_chttp2_hpack_compressor c;
   while (state.KeepRunning()) {
@@ -62,11 +61,13 @@ static void BM_HpackEncoderInitDestroy(benchmark::State &state) {
     grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_HpackEncoderInitDestroy);
 
 template <class Fixture>
 static void BM_HpackEncoderEncodeHeader(benchmark::State &state) {
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
 
   grpc_metadata_batch b;
@@ -102,6 +103,7 @@ static void BM_HpackEncoderEncodeHeader(benchmark::State &state) {
         << " header_bytes/iter:" << (static_cast<double>(stats.header_bytes) /
                                      static_cast<double>(state.iterations()));
   state.SetLabel(label.str());
+  track_counters.Finish(state);
 }
 
 namespace hpack_encoder_fixtures {
@@ -216,6 +218,7 @@ BENCHMARK_TEMPLATE(BM_HpackEncoderEncodeHeader,
 //
 
 static void BM_HpackParserInitDestroy(benchmark::State &state) {
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_chttp2_hpack_parser p;
   while (state.KeepRunning()) {
@@ -224,6 +227,7 @@ static void BM_HpackParserInitDestroy(benchmark::State &state) {
     grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_HpackParserInitDestroy);
 
@@ -234,6 +238,7 @@ static void UnrefHeader(grpc_exec_ctx *exec_ctx, void *user_data,
 
 template <class Fixture>
 static void BM_HpackParserParseHeader(benchmark::State &state) {
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   std::vector<grpc_slice> init_slices = Fixture::GetInitSlices();
   std::vector<grpc_slice> benchmark_slices = Fixture::GetBenchmarkSlices();
@@ -252,6 +257,7 @@ static void BM_HpackParserParseHeader(benchmark::State &state) {
   }
   grpc_chttp2_hpack_parser_destroy(&exec_ctx, &p);
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 
 namespace hpack_parser_fixtures {
diff --git a/test/cpp/microbenchmarks/bm_closure.cc b/test/cpp/microbenchmarks/bm_closure.cc
index 1f54e8c8b1..28a385b6c1 100644
--- a/test/cpp/microbenchmarks/bm_closure.cc
+++ b/test/cpp/microbenchmarks/bm_closure.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2017, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -42,86 +42,47 @@ extern "C" {
 #include "src/core/lib/support/spinlock.h"
 }
 
+#include "test/cpp/microbenchmarks/helpers.h"
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
-#include <sstream>
-
-#ifdef GPR_LOW_LEVEL_COUNTERS
-extern "C" gpr_atm gpr_mu_locks;
-#endif
-
-static class InitializeStuff {
- public:
-  InitializeStuff() { grpc_init(); }
-  ~InitializeStuff() { grpc_shutdown(); }
-} initialize_stuff;
-
-class TrackCounters {
- public:
-  TrackCounters(benchmark::State& state) : state_(state) {}
-
-  ~TrackCounters() {
-    std::ostringstream out;
-#ifdef GPR_LOW_LEVEL_COUNTERS
-    out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) -
-                                       mu_locks_at_start_) /
-                              (double)state_.iterations())
-        << " atm_cas/iter:"
-        << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) -
-                     atm_cas_at_start_) /
-            (double)state_.iterations())
-        << " atm_add/iter:"
-        << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) -
-                     atm_add_at_start_) /
-            (double)state_.iterations());
-#endif
-    state_.SetLabel(out.str());
-  }
-
- private:
-  benchmark::State& state_;
-#ifdef GPR_LOW_LEVEL_COUNTERS
-  const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks);
-  const size_t atm_cas_at_start_ =
-      gpr_atm_no_barrier_load(&gpr_counter_atm_cas);
-  const size_t atm_add_at_start_ =
-      gpr_atm_no_barrier_load(&gpr_counter_atm_add);
-#endif
-};
+auto& force_library_initialization = Library::get();
 
 static void BM_NoOpExecCtx(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
     grpc_exec_ctx_finish(&exec_ctx);
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_NoOpExecCtx);
 
 static void BM_WellFlushed(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   while (state.KeepRunning()) {
     grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_WellFlushed);
 
 static void DoNothing(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {}
 
 static void BM_ClosureInitAgainstExecCtx(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_closure c;
   while (state.KeepRunning()) {
     benchmark::DoNotOptimize(
         grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx));
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureInitAgainstExecCtx);
 
 static void BM_ClosureInitAgainstCombiner(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   grpc_closure c;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -131,11 +92,12 @@ static void BM_ClosureInitAgainstCombiner(benchmark::State& state) {
   }
   GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureInitAgainstCombiner);
 
 static void BM_ClosureRunOnExecCtx(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_closure c;
   grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -144,11 +106,12 @@ static void BM_ClosureRunOnExecCtx(benchmark::State& state) {
     grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureRunOnExecCtx);
 
 static void BM_ClosureCreateAndRun(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   while (state.KeepRunning()) {
     grpc_closure_run(&exec_ctx, grpc_closure_create(DoNothing, NULL,
@@ -156,11 +119,12 @@ static void BM_ClosureCreateAndRun(benchmark::State& state) {
                      GRPC_ERROR_NONE);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureCreateAndRun);
 
 static void BM_ClosureInitAndRun(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_closure c;
   while (state.KeepRunning()) {
@@ -169,11 +133,12 @@ static void BM_ClosureInitAndRun(benchmark::State& state) {
                      GRPC_ERROR_NONE);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureInitAndRun);
 
 static void BM_ClosureSchedOnExecCtx(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_closure c;
   grpc_closure_init(&c, DoNothing, NULL, grpc_schedule_on_exec_ctx);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -182,11 +147,12 @@ static void BM_ClosureSchedOnExecCtx(benchmark::State& state) {
     grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSchedOnExecCtx);
 
 static void BM_ClosureSched2OnExecCtx(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_closure c1;
   grpc_closure c2;
   grpc_closure_init(&c1, DoNothing, NULL, grpc_schedule_on_exec_ctx);
@@ -198,11 +164,12 @@ static void BM_ClosureSched2OnExecCtx(benchmark::State& state) {
     grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSched2OnExecCtx);
 
 static void BM_ClosureSched3OnExecCtx(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_closure c1;
   grpc_closure c2;
   grpc_closure c3;
@@ -217,11 +184,12 @@ static void BM_ClosureSched3OnExecCtx(benchmark::State& state) {
     grpc_exec_ctx_flush(&exec_ctx);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSched3OnExecCtx);
 
 static void BM_AcquireMutex(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   // for comparison with the combiner stuff below
   gpr_mu mu;
   gpr_mu_init(&mu);
@@ -232,11 +200,12 @@ static void BM_AcquireMutex(benchmark::State& state) {
     gpr_mu_unlock(&mu);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_AcquireMutex);
 
 static void BM_TryAcquireMutex(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   // for comparison with the combiner stuff below
   gpr_mu mu;
   gpr_mu_init(&mu);
@@ -250,11 +219,12 @@ static void BM_TryAcquireMutex(benchmark::State& state) {
     }
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_TryAcquireMutex);
 
 static void BM_AcquireSpinlock(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   // for comparison with the combiner stuff below
   gpr_spinlock mu = GPR_SPINLOCK_INITIALIZER;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -264,11 +234,12 @@ static void BM_AcquireSpinlock(benchmark::State& state) {
     gpr_spinlock_unlock(&mu);
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_AcquireSpinlock);
 
 static void BM_TryAcquireSpinlock(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   // for comparison with the combiner stuff below
   gpr_spinlock mu = GPR_SPINLOCK_INITIALIZER;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -281,11 +252,12 @@ static void BM_TryAcquireSpinlock(benchmark::State& state) {
     }
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_TryAcquireSpinlock);
 
 static void BM_ClosureSchedOnCombiner(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   grpc_closure c;
   grpc_closure_init(&c, DoNothing, NULL,
@@ -297,11 +269,12 @@ static void BM_ClosureSchedOnCombiner(benchmark::State& state) {
   }
   GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSchedOnCombiner);
 
 static void BM_ClosureSched2OnCombiner(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   grpc_closure c1;
   grpc_closure c2;
@@ -317,11 +290,12 @@ static void BM_ClosureSched2OnCombiner(benchmark::State& state) {
   }
   GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSched2OnCombiner);
 
 static void BM_ClosureSched3OnCombiner(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   grpc_closure c1;
   grpc_closure c2;
@@ -341,11 +315,12 @@ static void BM_ClosureSched3OnCombiner(benchmark::State& state) {
   }
   GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSched3OnCombiner);
 
 static void BM_ClosureSched2OnTwoCombiners(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_combiner* combiner1 = grpc_combiner_create(NULL);
   grpc_combiner* combiner2 = grpc_combiner_create(NULL);
   grpc_closure c1;
@@ -363,11 +338,12 @@ static void BM_ClosureSched2OnTwoCombiners(benchmark::State& state) {
   GRPC_COMBINER_UNREF(&exec_ctx, combiner1, "finished");
   GRPC_COMBINER_UNREF(&exec_ctx, combiner2, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSched2OnTwoCombiners);
 
 static void BM_ClosureSched4OnTwoCombiners(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_combiner* combiner1 = grpc_combiner_create(NULL);
   grpc_combiner* combiner2 = grpc_combiner_create(NULL);
   grpc_closure c1;
@@ -393,6 +369,7 @@ static void BM_ClosureSched4OnTwoCombiners(benchmark::State& state) {
   GRPC_COMBINER_UNREF(&exec_ctx, combiner1, "finished");
   GRPC_COMBINER_UNREF(&exec_ctx, combiner2, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureSched4OnTwoCombiners);
 
@@ -428,16 +405,17 @@ class Rescheduler {
 };
 
 static void BM_ClosureReschedOnExecCtx(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   Rescheduler r(state, grpc_schedule_on_exec_ctx);
   r.ScheduleFirst(&exec_ctx);
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureReschedOnExecCtx);
 
 static void BM_ClosureReschedOnCombiner(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   Rescheduler r(state, grpc_combiner_scheduler(combiner, false));
@@ -445,11 +423,12 @@ static void BM_ClosureReschedOnCombiner(benchmark::State& state) {
   grpc_exec_ctx_flush(&exec_ctx);
   GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureReschedOnCombiner);
 
 static void BM_ClosureReschedOnCombinerFinally(benchmark::State& state) {
-  TrackCounters track_counters(state);
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_combiner* combiner = grpc_combiner_create(NULL);
   Rescheduler r(state, grpc_combiner_finally_scheduler(combiner, false));
@@ -458,6 +437,7 @@ static void BM_ClosureReschedOnCombinerFinally(benchmark::State& state) {
   grpc_exec_ctx_flush(&exec_ctx);
   GRPC_COMBINER_UNREF(&exec_ctx, combiner, "finished");
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ClosureReschedOnCombinerFinally);
 
diff --git a/test/cpp/microbenchmarks/bm_cq.cc b/test/cpp/microbenchmarks/bm_cq.cc
index c017474bf4..91e6a85101 100644
--- a/test/cpp/microbenchmarks/bm_cq.cc
+++ b/test/cpp/microbenchmarks/bm_cq.cc
@@ -38,6 +38,7 @@
 #include <grpc++/impl/grpc_library.h>
 #include <grpc/grpc.h>
 
+#include "test/cpp/microbenchmarks/helpers.h"
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
 extern "C" {
@@ -47,27 +48,23 @@ extern "C" {
 namespace grpc {
 namespace testing {
 
-static class InitializeStuff {
- public:
-  InitializeStuff() { init_lib_.init(); }
-  ~InitializeStuff() { init_lib_.shutdown(); }
-
- private:
-  internal::GrpcLibrary init_lib_;
-  internal::GrpcLibraryInitializer init_;
-} initialize_stuff;
+auto& force_library_initialization = Library::get();
 
 static void BM_CreateDestroyCpp(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     CompletionQueue cq;
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_CreateDestroyCpp);
 
 static void BM_CreateDestroyCore(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     grpc_completion_queue_destroy(grpc_completion_queue_create(NULL));
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_CreateDestroyCore);
 
@@ -80,6 +77,7 @@ class DummyTag final : public CompletionQueueTag {
 };
 
 static void BM_Pass1Cpp(benchmark::State& state) {
+  TrackCounters track_counters;
   CompletionQueue cq;
   grpc_completion_queue* c_cq = cq.cq();
   while (state.KeepRunning()) {
@@ -94,10 +92,12 @@ static void BM_Pass1Cpp(benchmark::State& state) {
     bool ok;
     cq.Next(&tag, &ok);
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_Pass1Cpp);
 
 static void BM_Pass1Core(benchmark::State& state) {
+  TrackCounters track_counters;
   grpc_completion_queue* cq = grpc_completion_queue_create(NULL);
   gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
   while (state.KeepRunning()) {
@@ -110,10 +110,12 @@ static void BM_Pass1Core(benchmark::State& state) {
     grpc_completion_queue_next(cq, deadline, NULL);
   }
   grpc_completion_queue_destroy(cq);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_Pass1Core);
 
 static void BM_Pluck1Core(benchmark::State& state) {
+  TrackCounters track_counters;
   grpc_completion_queue* cq = grpc_completion_queue_create(NULL);
   gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
   while (state.KeepRunning()) {
@@ -126,16 +128,19 @@ static void BM_Pluck1Core(benchmark::State& state) {
     grpc_completion_queue_pluck(cq, NULL, deadline, NULL);
   }
   grpc_completion_queue_destroy(cq);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_Pluck1Core);
 
 static void BM_EmptyCore(benchmark::State& state) {
+  TrackCounters track_counters;
   grpc_completion_queue* cq = grpc_completion_queue_create(NULL);
   gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC);
   while (state.KeepRunning()) {
     grpc_completion_queue_next(cq, deadline, NULL);
   }
   grpc_completion_queue_destroy(cq);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_EmptyCore);
 
diff --git a/test/cpp/microbenchmarks/bm_error.cc b/test/cpp/microbenchmarks/bm_error.cc
index 8a4b86f281..95f19e7586 100644
--- a/test/cpp/microbenchmarks/bm_error.cc
+++ b/test/cpp/microbenchmarks/bm_error.cc
@@ -40,8 +40,11 @@ extern "C" {
 #include "src/core/lib/transport/error_utils.h"
 }
 
+#include "test/cpp/microbenchmarks/helpers.h"
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
+auto& force_library_initialization = Library::get();
+
 class ErrorDeleter {
  public:
   void operator()(grpc_error* error) { GRPC_ERROR_UNREF(error); }
@@ -49,31 +52,38 @@ class ErrorDeleter {
 typedef std::unique_ptr<grpc_error, ErrorDeleter> ErrorPtr;
 
 static void BM_ErrorCreate(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     GRPC_ERROR_UNREF(GRPC_ERROR_CREATE("Error"));
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ErrorCreate);
 
 static void BM_ErrorCreateAndSetStatus(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     GRPC_ERROR_UNREF(grpc_error_set_int(GRPC_ERROR_CREATE("Error"),
                                         GRPC_ERROR_INT_GRPC_STATUS,
                                         GRPC_STATUS_ABORTED));
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ErrorCreateAndSetStatus);
 
 static void BM_ErrorRefUnref(benchmark::State& state) {
+  TrackCounters track_counters;
   grpc_error* error = GRPC_ERROR_CREATE("Error");
   while (state.KeepRunning()) {
     GRPC_ERROR_UNREF(GRPC_ERROR_REF(error));
   }
   GRPC_ERROR_UNREF(error);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ErrorRefUnref);
 
 static void BM_ErrorUnrefNone(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     GRPC_ERROR_UNREF(GRPC_ERROR_NONE);
   }
@@ -81,30 +91,36 @@ static void BM_ErrorUnrefNone(benchmark::State& state) {
 BENCHMARK(BM_ErrorUnrefNone);
 
 static void BM_ErrorGetIntFromNoError(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     intptr_t value;
     grpc_error_get_int(GRPC_ERROR_NONE, GRPC_ERROR_INT_GRPC_STATUS, &value);
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ErrorGetIntFromNoError);
 
 static void BM_ErrorGetMissingInt(benchmark::State& state) {
+  TrackCounters track_counters;
   ErrorPtr error(
       grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_INDEX, 1));
   while (state.KeepRunning()) {
     intptr_t value;
     grpc_error_get_int(error.get(), GRPC_ERROR_INT_OFFSET, &value);
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ErrorGetMissingInt);
 
 static void BM_ErrorGetPresentInt(benchmark::State& state) {
+  TrackCounters track_counters;
   ErrorPtr error(
       grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_OFFSET, 1));
   while (state.KeepRunning()) {
     intptr_t value;
     grpc_error_get_int(error.get(), GRPC_ERROR_INT_OFFSET, &value);
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_ErrorGetPresentInt);
 
@@ -177,22 +193,27 @@ class ErrorWithNestedGrpcStatus {
 
 template <class Fixture>
 static void BM_ErrorStringOnNewError(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     Fixture fixture;
     grpc_error_string(fixture.error());
   }
+  track_counters.Finish(state);
 }
 
 template <class Fixture>
 static void BM_ErrorStringRepeatedly(benchmark::State& state) {
+  TrackCounters track_counters;
   Fixture fixture;
   while (state.KeepRunning()) {
     grpc_error_string(fixture.error());
   }
+  track_counters.Finish(state);
 }
 
 template <class Fixture>
 static void BM_ErrorGetStatus(benchmark::State& state) {
+  TrackCounters track_counters;
   Fixture fixture;
   while (state.KeepRunning()) {
     grpc_status_code status;
@@ -200,34 +221,41 @@ static void BM_ErrorGetStatus(benchmark::State& state) {
     grpc_error_get_status(fixture.error(), fixture.deadline(), &status, &msg,
                           NULL);
   }
+  track_counters.Finish(state);
 }
 
 template <class Fixture>
 static void BM_ErrorGetStatusCode(benchmark::State& state) {
+  TrackCounters track_counters;
   Fixture fixture;
   while (state.KeepRunning()) {
     grpc_status_code status;
     grpc_error_get_status(fixture.error(), fixture.deadline(), &status, NULL,
                           NULL);
   }
+  track_counters.Finish(state);
 }
 
 template <class Fixture>
 static void BM_ErrorHttpError(benchmark::State& state) {
+  TrackCounters track_counters;
   Fixture fixture;
   while (state.KeepRunning()) {
     grpc_http2_error_code error;
     grpc_error_get_status(fixture.error(), fixture.deadline(), NULL, NULL,
                           &error);
   }
+  track_counters.Finish(state);
 }
 
 template <class Fixture>
 static void BM_HasClearGrpcStatus(benchmark::State& state) {
+  TrackCounters track_counters;
   Fixture fixture;
   while (state.KeepRunning()) {
     grpc_error_has_clear_grpc_status(fixture.error());
   }
+  track_counters.Finish(state);
 }
 
 #define BENCHMARK_SUITE(fixture)                         \
diff --git a/test/cpp/microbenchmarks/bm_fullstack.cc b/test/cpp/microbenchmarks/bm_fullstack.cc
deleted file mode 100644
index 48e131f1be..0000000000
--- a/test/cpp/microbenchmarks/bm_fullstack.cc
+++ /dev/null
@@ -1,1079 +0,0 @@
-/*
- *
- * Copyright 2016, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-/* Benchmark gRPC end2end in various configurations */
-
-#include <sstream>
-
-#include <grpc++/channel.h>
-#include <grpc++/create_channel.h>
-#include <grpc++/impl/grpc_library.h>
-#include <grpc++/security/credentials.h>
-#include <grpc++/security/server_credentials.h>
-#include <grpc++/server.h>
-#include <grpc++/server_builder.h>
-#include <grpc/support/log.h>
-
-extern "C" {
-#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
-#include "src/core/ext/transport/chttp2/transport/internal.h"
-#include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/iomgr/endpoint.h"
-#include "src/core/lib/iomgr/endpoint_pair.h"
-#include "src/core/lib/iomgr/exec_ctx.h"
-#include "src/core/lib/iomgr/tcp_posix.h"
-#include "src/core/lib/surface/channel.h"
-#include "src/core/lib/surface/completion_queue.h"
-#include "src/core/lib/surface/server.h"
-#include "test/core/util/memory_counters.h"
-#include "test/core/util/passthru_endpoint.h"
-#include "test/core/util/port.h"
-#include "test/core/util/trickle_endpoint.h"
-}
-#include "src/core/lib/profiling/timers.h"
-#include "src/cpp/client/create_channel_internal.h"
-#include "src/proto/grpc/testing/echo.grpc.pb.h"
-#include "third_party/benchmark/include/benchmark/benchmark.h"
-
-namespace grpc {
-namespace testing {
-
-static class InitializeStuff {
- public:
-  InitializeStuff() {
-    grpc_memory_counters_init();
-    init_lib_.init();
-    rq_ = grpc_resource_quota_create("bm");
-  }
-
-  ~InitializeStuff() { init_lib_.shutdown(); }
-
-  grpc_resource_quota* rq() { return rq_; }
-
- private:
-  internal::GrpcLibrary init_lib_;
-  grpc_resource_quota* rq_;
-} initialize_stuff;
-
-/*******************************************************************************
- * FIXTURES
- */
-
-static void ApplyCommonServerBuilderConfig(ServerBuilder* b) {
-  b->SetMaxReceiveMessageSize(INT_MAX);
-  b->SetMaxSendMessageSize(INT_MAX);
-}
-
-static void ApplyCommonChannelArguments(ChannelArguments* c) {
-  c->SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, INT_MAX);
-  c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX);
-}
-
-#ifdef GPR_LOW_LEVEL_COUNTERS
-extern "C" gpr_atm gpr_mu_locks;
-extern "C" gpr_atm gpr_counter_atm_cas;
-extern "C" gpr_atm gpr_counter_atm_add;
-#endif
-
-class BaseFixture {
- public:
-  void Finish(benchmark::State& s) {
-    std::ostringstream out;
-    this->AddToLabel(out, s);
-#ifdef GPR_LOW_LEVEL_COUNTERS
-    out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) -
-                                       mu_locks_at_start_) /
-                              (double)s.iterations())
-        << " atm_cas/iter:"
-        << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) -
-                     atm_cas_at_start_) /
-            (double)s.iterations())
-        << " atm_add/iter:"
-        << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) -
-                     atm_add_at_start_) /
-            (double)s.iterations());
-#endif
-    grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot();
-    out << " allocs/iter:"
-        << ((double)(counters_at_end.total_allocs_absolute -
-                     counters_at_start_.total_allocs_absolute) /
-            (double)s.iterations());
-    auto label = out.str();
-    if (label.length() && label[0] == ' ') {
-      label = label.substr(1);
-    }
-    s.SetLabel(label);
-  }
-
-  virtual void AddToLabel(std::ostream& out, benchmark::State& s) = 0;
-
- private:
-#ifdef GPR_LOW_LEVEL_COUNTERS
-  const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks);
-  const size_t atm_cas_at_start_ =
-      gpr_atm_no_barrier_load(&gpr_counter_atm_cas);
-  const size_t atm_add_at_start_ =
-      gpr_atm_no_barrier_load(&gpr_counter_atm_add);
-#endif
-  grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot();
-};
-
-class FullstackFixture : public BaseFixture {
- public:
-  FullstackFixture(Service* service, const grpc::string& address) {
-    ServerBuilder b;
-    b.AddListeningPort(address, InsecureServerCredentials());
-    cq_ = b.AddCompletionQueue(true);
-    b.RegisterService(service);
-    ApplyCommonServerBuilderConfig(&b);
-    server_ = b.BuildAndStart();
-    ChannelArguments args;
-    ApplyCommonChannelArguments(&args);
-    channel_ = CreateCustomChannel(address, InsecureChannelCredentials(), args);
-  }
-
-  virtual ~FullstackFixture() {
-    server_->Shutdown();
-    cq_->Shutdown();
-    void* tag;
-    bool ok;
-    while (cq_->Next(&tag, &ok)) {
-    }
-  }
-
-  ServerCompletionQueue* cq() { return cq_.get(); }
-  std::shared_ptr<Channel> channel() { return channel_; }
-
- private:
-  std::unique_ptr<Server> server_;
-  std::unique_ptr<ServerCompletionQueue> cq_;
-  std::shared_ptr<Channel> channel_;
-};
-
-class TCP : public FullstackFixture {
- public:
-  TCP(Service* service) : FullstackFixture(service, MakeAddress()) {}
-
-  void AddToLabel(std::ostream& out, benchmark::State& state) {}
-
- private:
-  static grpc::string MakeAddress() {
-    int port = grpc_pick_unused_port_or_die();
-    std::stringstream addr;
-    addr << "localhost:" << port;
-    return addr.str();
-  }
-};
-
-class UDS : public FullstackFixture {
- public:
-  UDS(Service* service) : FullstackFixture(service, MakeAddress()) {}
-
-  void AddToLabel(std::ostream& out, benchmark::State& state) override {}
-
- private:
-  static grpc::string MakeAddress() {
-    int port = grpc_pick_unused_port_or_die();  // just for a unique id - not a
-                                                // real port
-    std::stringstream addr;
-    addr << "unix:/tmp/bm_fullstack." << port;
-    return addr.str();
-  }
-};
-
-class EndpointPairFixture : public BaseFixture {
- public:
-  EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints)
-      : endpoint_pair_(endpoints) {
-    ServerBuilder b;
-    cq_ = b.AddCompletionQueue(true);
-    b.RegisterService(service);
-    ApplyCommonServerBuilderConfig(&b);
-    server_ = b.BuildAndStart();
-
-    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-
-    /* add server endpoint to server_ */
-    {
-      const grpc_channel_args* server_args =
-          grpc_server_get_channel_args(server_->c_server());
-      server_transport_ = grpc_create_chttp2_transport(
-          &exec_ctx, server_args, endpoints.server, 0 /* is_client */);
-
-      grpc_pollset** pollsets;
-      size_t num_pollsets = 0;
-      grpc_server_get_pollsets(server_->c_server(), &pollsets, &num_pollsets);
-
-      for (size_t i = 0; i < num_pollsets; i++) {
-        grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]);
-      }
-
-      grpc_server_setup_transport(&exec_ctx, server_->c_server(),
-                                  server_transport_, NULL, server_args);
-      grpc_chttp2_transport_start_reading(&exec_ctx, server_transport_, NULL);
-    }
-
-    /* create channel */
-    {
-      ChannelArguments args;
-      args.SetString(GRPC_ARG_DEFAULT_AUTHORITY, "test.authority");
-      ApplyCommonChannelArguments(&args);
-
-      grpc_channel_args c_args = args.c_channel_args();
-      client_transport_ =
-          grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1);
-      GPR_ASSERT(client_transport_);
-      grpc_channel* channel =
-          grpc_channel_create(&exec_ctx, "target", &c_args,
-                              GRPC_CLIENT_DIRECT_CHANNEL, client_transport_);
-      grpc_chttp2_transport_start_reading(&exec_ctx, client_transport_, NULL);
-
-      channel_ = CreateChannelInternal("", channel);
-    }
-
-    grpc_exec_ctx_finish(&exec_ctx);
-  }
-
-  virtual ~EndpointPairFixture() {
-    server_->Shutdown();
-    cq_->Shutdown();
-    void* tag;
-    bool ok;
-    while (cq_->Next(&tag, &ok)) {
-    }
-  }
-
-  ServerCompletionQueue* cq() { return cq_.get(); }
-  std::shared_ptr<Channel> channel() { return channel_; }
-
- protected:
-  grpc_endpoint_pair endpoint_pair_;
-  grpc_transport* client_transport_;
-  grpc_transport* server_transport_;
-
- private:
-  std::unique_ptr<Server> server_;
-  std::unique_ptr<ServerCompletionQueue> cq_;
-  std::shared_ptr<Channel> channel_;
-};
-
-class SockPair : public EndpointPairFixture {
- public:
-  SockPair(Service* service)
-      : EndpointPairFixture(service, grpc_iomgr_create_endpoint_pair(
-                                         "test", initialize_stuff.rq(), 8192)) {
-  }
-
-  void AddToLabel(std::ostream& out, benchmark::State& state) {}
-};
-
-class InProcessCHTTP2 : public EndpointPairFixture {
- public:
-  InProcessCHTTP2(Service* service)
-      : EndpointPairFixture(service, MakeEndpoints()) {}
-
-  void AddToLabel(std::ostream& out, benchmark::State& state) {
-    out << " writes/iter:"
-        << ((double)stats_.num_writes / (double)state.iterations());
-  }
-
- private:
-  grpc_passthru_endpoint_stats stats_;
-
-  grpc_endpoint_pair MakeEndpoints() {
-    grpc_endpoint_pair p;
-    grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(),
-                                  &stats_);
-    return p;
-  }
-};
-
-class TrickledCHTTP2 : public EndpointPairFixture {
- public:
-  TrickledCHTTP2(Service* service, size_t megabits_per_second)
-      : EndpointPairFixture(service, MakeEndpoints(megabits_per_second)) {}
-
-  void AddToLabel(std::ostream& out, benchmark::State& state) {
-    out << " writes/iter:"
-        << ((double)stats_.num_writes / (double)state.iterations())
-        << " cli_transport_stalls/iter:"
-        << ((double)
-                client_stats_.streams_stalled_due_to_transport_flow_control /
-            (double)state.iterations())
-        << " cli_stream_stalls/iter:"
-        << ((double)client_stats_.streams_stalled_due_to_stream_flow_control /
-            (double)state.iterations())
-        << " svr_transport_stalls/iter:"
-        << ((double)
-                server_stats_.streams_stalled_due_to_transport_flow_control /
-            (double)state.iterations())
-        << " svr_stream_stalls/iter:"
-        << ((double)server_stats_.streams_stalled_due_to_stream_flow_control /
-            (double)state.iterations());
-  }
-
-  void Step() {
-    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-    size_t client_backlog =
-        grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.client);
-    size_t server_backlog =
-        grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.server);
-    grpc_exec_ctx_finish(&exec_ctx);
-
-    UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_,
-                client_backlog);
-    UpdateStats((grpc_chttp2_transport*)server_transport_, &server_stats_,
-                server_backlog);
-  }
-
- private:
-  grpc_passthru_endpoint_stats stats_;
-  struct Stats {
-    int streams_stalled_due_to_stream_flow_control = 0;
-    int streams_stalled_due_to_transport_flow_control = 0;
-  };
-  Stats client_stats_;
-  Stats server_stats_;
-
-  grpc_endpoint_pair MakeEndpoints(size_t kilobits) {
-    grpc_endpoint_pair p;
-    grpc_passthru_endpoint_create(&p.client, &p.server, initialize_stuff.rq(),
-                                  &stats_);
-    double bytes_per_second = 125.0 * kilobits;
-    p.client = grpc_trickle_endpoint_create(p.client, bytes_per_second);
-    p.server = grpc_trickle_endpoint_create(p.server, bytes_per_second);
-    return p;
-  }
-
-  void UpdateStats(grpc_chttp2_transport* t, Stats* s, size_t backlog) {
-    if (backlog == 0) {
-      if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_STREAM].head != NULL) {
-        s->streams_stalled_due_to_stream_flow_control++;
-      }
-      if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT].head != NULL) {
-        s->streams_stalled_due_to_transport_flow_control++;
-      }
-    }
-  }
-};
-
-/*******************************************************************************
- * CONTEXT MUTATORS
- */
-
-static const int kPregenerateKeyCount = 100000;
-
-template <class F>
-auto MakeVector(size_t length, F f) -> std::vector<decltype(f())> {
-  std::vector<decltype(f())> out;
-  out.reserve(length);
-  for (size_t i = 0; i < length; i++) {
-    out.push_back(f());
-  }
-  return out;
-}
-
-class NoOpMutator {
- public:
-  template <class ContextType>
-  NoOpMutator(ContextType* context) {}
-};
-
-template <int length>
-class RandomBinaryMetadata {
- public:
-  static const grpc::string& Key() { return kKey; }
-
-  static const grpc::string& Value() {
-    return kValues[rand() % kValues.size()];
-  }
-
- private:
-  static const grpc::string kKey;
-  static const std::vector<grpc::string> kValues;
-
-  static grpc::string GenerateOneString() {
-    grpc::string s;
-    s.reserve(length + 1);
-    for (int i = 0; i < length; i++) {
-      s += (char)rand();
-    }
-    return s;
-  }
-};
-
-template <int length>
-const grpc::string RandomBinaryMetadata<length>::kKey = "foo-bin";
-
-template <int length>
-const std::vector<grpc::string> RandomBinaryMetadata<length>::kValues =
-    MakeVector(kPregenerateKeyCount, GenerateOneString);
-
-template <int length>
-class RandomAsciiMetadata {
- public:
-  static const grpc::string& Key() { return kKey; }
-
-  static const grpc::string& Value() {
-    return kValues[rand() % kValues.size()];
-  }
-
- private:
-  static const grpc::string kKey;
-  static const std::vector<grpc::string> kValues;
-
-  static grpc::string GenerateOneString() {
-    grpc::string s;
-    s.reserve(length + 1);
-    for (int i = 0; i < length; i++) {
-      s += (char)(rand() % 26 + 'a');
-    }
-    return s;
-  }
-};
-
-template <int length>
-const grpc::string RandomAsciiMetadata<length>::kKey = "foo";
-
-template <int length>
-const std::vector<grpc::string> RandomAsciiMetadata<length>::kValues =
-    MakeVector(kPregenerateKeyCount, GenerateOneString);
-
-template <class Generator, int kNumKeys>
-class Client_AddMetadata : public NoOpMutator {
- public:
-  Client_AddMetadata(ClientContext* context) : NoOpMutator(context) {
-    for (int i = 0; i < kNumKeys; i++) {
-      context->AddMetadata(Generator::Key(), Generator::Value());
-    }
-  }
-};
-
-template <class Generator, int kNumKeys>
-class Server_AddInitialMetadata : public NoOpMutator {
- public:
-  Server_AddInitialMetadata(ServerContext* context) : NoOpMutator(context) {
-    for (int i = 0; i < kNumKeys; i++) {
-      context->AddInitialMetadata(Generator::Key(), Generator::Value());
-    }
-  }
-};
-
-/*******************************************************************************
- * BENCHMARKING KERNELS
- */
-
-static void* tag(intptr_t x) { return reinterpret_cast<void*>(x); }
-
-template <class Fixture, class ClientContextMutator, class ServerContextMutator>
-static void BM_UnaryPingPong(benchmark::State& state) {
-  EchoTestService::AsyncService service;
-  std::unique_ptr<Fixture> fixture(new Fixture(&service));
-  EchoRequest send_request;
-  EchoResponse send_response;
-  EchoResponse recv_response;
-  if (state.range(0) > 0) {
-    send_request.set_message(std::string(state.range(0), 'a'));
-  }
-  if (state.range(1) > 0) {
-    send_response.set_message(std::string(state.range(1), 'a'));
-  }
-  Status recv_status;
-  struct ServerEnv {
-    ServerContext ctx;
-    EchoRequest recv_request;
-    grpc::ServerAsyncResponseWriter<EchoResponse> response_writer;
-    ServerEnv() : response_writer(&ctx) {}
-  };
-  uint8_t server_env_buffer[2 * sizeof(ServerEnv)];
-  ServerEnv* server_env[2] = {
-      reinterpret_cast<ServerEnv*>(server_env_buffer),
-      reinterpret_cast<ServerEnv*>(server_env_buffer + sizeof(ServerEnv))};
-  new (server_env[0]) ServerEnv;
-  new (server_env[1]) ServerEnv;
-  service.RequestEcho(&server_env[0]->ctx, &server_env[0]->recv_request,
-                      &server_env[0]->response_writer, fixture->cq(),
-                      fixture->cq(), tag(0));
-  service.RequestEcho(&server_env[1]->ctx, &server_env[1]->recv_request,
-                      &server_env[1]->response_writer, fixture->cq(),
-                      fixture->cq(), tag(1));
-  std::unique_ptr<EchoTestService::Stub> stub(
-      EchoTestService::NewStub(fixture->channel()));
-  while (state.KeepRunning()) {
-    GPR_TIMER_SCOPE("BenchmarkCycle", 0);
-    recv_response.Clear();
-    ClientContext cli_ctx;
-    ClientContextMutator cli_ctx_mut(&cli_ctx);
-    std::unique_ptr<ClientAsyncResponseReader<EchoResponse>> response_reader(
-        stub->AsyncEcho(&cli_ctx, send_request, fixture->cq()));
-    void* t;
-    bool ok;
-    GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-    GPR_ASSERT(ok);
-    GPR_ASSERT(t == tag(0) || t == tag(1));
-    intptr_t slot = reinterpret_cast<intptr_t>(t);
-    ServerEnv* senv = server_env[slot];
-    ServerContextMutator svr_ctx_mut(&senv->ctx);
-    senv->response_writer.Finish(send_response, Status::OK, tag(3));
-    response_reader->Finish(&recv_response, &recv_status, tag(4));
-    for (int i = (1 << 3) | (1 << 4); i != 0;) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      GPR_ASSERT(ok);
-      int tagnum = (int)reinterpret_cast<intptr_t>(t);
-      GPR_ASSERT(i & (1 << tagnum));
-      i -= 1 << tagnum;
-    }
-    GPR_ASSERT(recv_status.ok());
-
-    senv->~ServerEnv();
-    senv = new (senv) ServerEnv();
-    service.RequestEcho(&senv->ctx, &senv->recv_request, &senv->response_writer,
-                        fixture->cq(), fixture->cq(), tag(slot));
-  }
-  fixture->Finish(state);
-  fixture.reset();
-  server_env[0]->~ServerEnv();
-  server_env[1]->~ServerEnv();
-  state.SetBytesProcessed(state.range(0) * state.iterations() +
-                          state.range(1) * state.iterations());
-}
-
-// Repeatedly makes Streaming Bidi calls (exchanging a configurable number of
-// messages in each call) in a loop on a single channel
-//
-//  First parmeter (i.e state.range(0)):  Message size (in bytes) to use
-//  Second parameter (i.e state.range(1)): Number of ping pong messages.
-//      Note: One ping-pong means two messages (one from client to server and
-//      the other from server to client):
-template <class Fixture, class ClientContextMutator, class ServerContextMutator>
-static void BM_StreamingPingPong(benchmark::State& state) {
-  const int msg_size = state.range(0);
-  const int max_ping_pongs = state.range(1);
-
-  EchoTestService::AsyncService service;
-  std::unique_ptr<Fixture> fixture(new Fixture(&service));
-  {
-    EchoResponse send_response;
-    EchoResponse recv_response;
-    EchoRequest send_request;
-    EchoRequest recv_request;
-
-    if (msg_size > 0) {
-      send_request.set_message(std::string(msg_size, 'a'));
-      send_response.set_message(std::string(msg_size, 'b'));
-    }
-
-    std::unique_ptr<EchoTestService::Stub> stub(
-        EchoTestService::NewStub(fixture->channel()));
-
-    while (state.KeepRunning()) {
-      ServerContext svr_ctx;
-      ServerContextMutator svr_ctx_mut(&svr_ctx);
-      ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
-      service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
-                                fixture->cq(), tag(0));
-
-      ClientContext cli_ctx;
-      ClientContextMutator cli_ctx_mut(&cli_ctx);
-      auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
-
-      // Establish async stream between client side and server side
-      void* t;
-      bool ok;
-      int need_tags = (1 << 0) | (1 << 1);
-      while (need_tags) {
-        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-        GPR_ASSERT(ok);
-        int i = (int)(intptr_t)t;
-        GPR_ASSERT(need_tags & (1 << i));
-        need_tags &= ~(1 << i);
-      }
-
-      // Send 'max_ping_pongs' number of ping pong messages
-      int ping_pong_cnt = 0;
-      while (ping_pong_cnt < max_ping_pongs) {
-        request_rw->Write(send_request, tag(0));   // Start client send
-        response_rw.Read(&recv_request, tag(1));   // Start server recv
-        request_rw->Read(&recv_response, tag(2));  // Start client recv
-
-        need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3);
-        while (need_tags) {
-          GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-          GPR_ASSERT(ok);
-          int i = (int)(intptr_t)t;
-
-          // If server recv is complete, start the server send operation
-          if (i == 1) {
-            response_rw.Write(send_response, tag(3));
-          }
-
-          GPR_ASSERT(need_tags & (1 << i));
-          need_tags &= ~(1 << i);
-        }
-
-        ping_pong_cnt++;
-      }
-
-      request_rw->WritesDone(tag(0));
-      response_rw.Finish(Status::OK, tag(1));
-
-      Status recv_status;
-      request_rw->Finish(&recv_status, tag(2));
-
-      need_tags = (1 << 0) | (1 << 1) | (1 << 2);
-      while (need_tags) {
-        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-        int i = (int)(intptr_t)t;
-        GPR_ASSERT(need_tags & (1 << i));
-        need_tags &= ~(1 << i);
-      }
-
-      GPR_ASSERT(recv_status.ok());
-    }
-  }
-
-  fixture->Finish(state);
-  fixture.reset();
-  state.SetBytesProcessed(msg_size * state.iterations() * max_ping_pongs * 2);
-}
-
-// Repeatedly sends ping pong messages in a single streaming Bidi call in a loop
-//     First parmeter (i.e state.range(0)):  Message size (in bytes) to use
-template <class Fixture, class ClientContextMutator, class ServerContextMutator>
-static void BM_StreamingPingPongMsgs(benchmark::State& state) {
-  const int msg_size = state.range(0);
-
-  EchoTestService::AsyncService service;
-  std::unique_ptr<Fixture> fixture(new Fixture(&service));
-  {
-    EchoResponse send_response;
-    EchoResponse recv_response;
-    EchoRequest send_request;
-    EchoRequest recv_request;
-
-    if (msg_size > 0) {
-      send_request.set_message(std::string(msg_size, 'a'));
-      send_response.set_message(std::string(msg_size, 'b'));
-    }
-
-    std::unique_ptr<EchoTestService::Stub> stub(
-        EchoTestService::NewStub(fixture->channel()));
-
-    ServerContext svr_ctx;
-    ServerContextMutator svr_ctx_mut(&svr_ctx);
-    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
-    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
-                              fixture->cq(), tag(0));
-
-    ClientContext cli_ctx;
-    ClientContextMutator cli_ctx_mut(&cli_ctx);
-    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
-
-    // Establish async stream between client side and server side
-    void* t;
-    bool ok;
-    int need_tags = (1 << 0) | (1 << 1);
-    while (need_tags) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      GPR_ASSERT(ok);
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-
-    while (state.KeepRunning()) {
-      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
-      request_rw->Write(send_request, tag(0));   // Start client send
-      response_rw.Read(&recv_request, tag(1));   // Start server recv
-      request_rw->Read(&recv_response, tag(2));  // Start client recv
-
-      need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3);
-      while (need_tags) {
-        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-        GPR_ASSERT(ok);
-        int i = (int)(intptr_t)t;
-
-        // If server recv is complete, start the server send operation
-        if (i == 1) {
-          response_rw.Write(send_response, tag(3));
-        }
-
-        GPR_ASSERT(need_tags & (1 << i));
-        need_tags &= ~(1 << i);
-      }
-    }
-
-    request_rw->WritesDone(tag(0));
-    response_rw.Finish(Status::OK, tag(1));
-    Status recv_status;
-    request_rw->Finish(&recv_status, tag(2));
-
-    need_tags = (1 << 0) | (1 << 1) | (1 << 2);
-    while (need_tags) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-
-    GPR_ASSERT(recv_status.ok());
-  }
-
-  fixture->Finish(state);
-  fixture.reset();
-  state.SetBytesProcessed(msg_size * state.iterations() * 2);
-}
-
-template <class Fixture>
-static void BM_PumpStreamClientToServer(benchmark::State& state) {
-  EchoTestService::AsyncService service;
-  std::unique_ptr<Fixture> fixture(new Fixture(&service));
-  {
-    EchoRequest send_request;
-    EchoRequest recv_request;
-    if (state.range(0) > 0) {
-      send_request.set_message(std::string(state.range(0), 'a'));
-    }
-    Status recv_status;
-    ServerContext svr_ctx;
-    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
-    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
-                              fixture->cq(), tag(0));
-    std::unique_ptr<EchoTestService::Stub> stub(
-        EchoTestService::NewStub(fixture->channel()));
-    ClientContext cli_ctx;
-    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
-    int need_tags = (1 << 0) | (1 << 1);
-    void* t;
-    bool ok;
-    while (need_tags) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      GPR_ASSERT(ok);
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-    response_rw.Read(&recv_request, tag(0));
-    while (state.KeepRunning()) {
-      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
-      request_rw->Write(send_request, tag(1));
-      while (true) {
-        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-        if (t == tag(0)) {
-          response_rw.Read(&recv_request, tag(0));
-        } else if (t == tag(1)) {
-          break;
-        } else {
-          GPR_ASSERT(false);
-        }
-      }
-    }
-    request_rw->WritesDone(tag(1));
-    need_tags = (1 << 0) | (1 << 1);
-    while (need_tags) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-  }
-  fixture->Finish(state);
-  fixture.reset();
-  state.SetBytesProcessed(state.range(0) * state.iterations());
-}
-
-template <class Fixture>
-static void BM_PumpStreamServerToClient(benchmark::State& state) {
-  EchoTestService::AsyncService service;
-  std::unique_ptr<Fixture> fixture(new Fixture(&service));
-  {
-    EchoResponse send_response;
-    EchoResponse recv_response;
-    if (state.range(0) > 0) {
-      send_response.set_message(std::string(state.range(0), 'a'));
-    }
-    Status recv_status;
-    ServerContext svr_ctx;
-    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
-    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
-                              fixture->cq(), tag(0));
-    std::unique_ptr<EchoTestService::Stub> stub(
-        EchoTestService::NewStub(fixture->channel()));
-    ClientContext cli_ctx;
-    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
-    int need_tags = (1 << 0) | (1 << 1);
-    void* t;
-    bool ok;
-    while (need_tags) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      GPR_ASSERT(ok);
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-    request_rw->Read(&recv_response, tag(0));
-    while (state.KeepRunning()) {
-      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
-      response_rw.Write(send_response, tag(1));
-      while (true) {
-        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-        if (t == tag(0)) {
-          request_rw->Read(&recv_response, tag(0));
-        } else if (t == tag(1)) {
-          break;
-        } else {
-          GPR_ASSERT(false);
-        }
-      }
-    }
-    response_rw.Finish(Status::OK, tag(1));
-    need_tags = (1 << 0) | (1 << 1);
-    while (need_tags) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-  }
-  fixture->Finish(state);
-  fixture.reset();
-  state.SetBytesProcessed(state.range(0) * state.iterations());
-}
-
-static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok) {
-  while (true) {
-    switch (fixture->cq()->AsyncNext(
-        t, ok, gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
-                            gpr_time_from_micros(100, GPR_TIMESPAN)))) {
-      case CompletionQueue::TIMEOUT:
-        fixture->Step();
-        break;
-      case CompletionQueue::SHUTDOWN:
-        GPR_ASSERT(false);
-        break;
-      case CompletionQueue::GOT_EVENT:
-        return;
-    }
-  }
-}
-
-static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) {
-  EchoTestService::AsyncService service;
-  std::unique_ptr<TrickledCHTTP2> fixture(
-      new TrickledCHTTP2(&service, state.range(1)));
-  {
-    EchoResponse send_response;
-    EchoResponse recv_response;
-    if (state.range(0) > 0) {
-      send_response.set_message(std::string(state.range(0), 'a'));
-    }
-    Status recv_status;
-    ServerContext svr_ctx;
-    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
-    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
-                              fixture->cq(), tag(0));
-    std::unique_ptr<EchoTestService::Stub> stub(
-        EchoTestService::NewStub(fixture->channel()));
-    ClientContext cli_ctx;
-    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
-    int need_tags = (1 << 0) | (1 << 1);
-    void* t;
-    bool ok;
-    while (need_tags) {
-      TrickleCQNext(fixture.get(), &t, &ok);
-      GPR_ASSERT(ok);
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-    request_rw->Read(&recv_response, tag(0));
-    while (state.KeepRunning()) {
-      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
-      response_rw.Write(send_response, tag(1));
-      while (true) {
-        TrickleCQNext(fixture.get(), &t, &ok);
-        if (t == tag(0)) {
-          request_rw->Read(&recv_response, tag(0));
-        } else if (t == tag(1)) {
-          break;
-        } else {
-          GPR_ASSERT(false);
-        }
-      }
-    }
-    response_rw.Finish(Status::OK, tag(1));
-    need_tags = (1 << 0) | (1 << 1);
-    while (need_tags) {
-      TrickleCQNext(fixture.get(), &t, &ok);
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-  }
-  fixture->Finish(state);
-  fixture.reset();
-  state.SetBytesProcessed(state.range(0) * state.iterations());
-}
-
-/*******************************************************************************
- * CONFIGURATIONS
- */
-
-static void SweepSizesArgs(benchmark::internal::Benchmark* b) {
-  b->Args({0, 0});
-  for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) {
-    b->Args({i, 0});
-    b->Args({0, i});
-    b->Args({i, i});
-  }
-}
-
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, TCP, NoOpMutator, NoOpMutator)
-    ->Apply(SweepSizesArgs);
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, UDS, NoOpMutator, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, SockPair, NoOpMutator, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, NoOpMutator)
-    ->Apply(SweepSizesArgs);
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomBinaryMetadata<10>, 1>, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomBinaryMetadata<31>, 1>, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomBinaryMetadata<100>, 1>,
-                   NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomBinaryMetadata<10>, 2>, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomBinaryMetadata<31>, 2>, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomBinaryMetadata<100>, 2>,
-                   NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
-                   Server_AddInitialMetadata<RandomBinaryMetadata<10>, 1>)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
-                   Server_AddInitialMetadata<RandomBinaryMetadata<31>, 1>)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
-                   Server_AddInitialMetadata<RandomBinaryMetadata<100>, 1>)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomAsciiMetadata<10>, 1>, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomAsciiMetadata<31>, 1>, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
-                   Client_AddMetadata<RandomAsciiMetadata<100>, 1>, NoOpMutator)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
-                   Server_AddInitialMetadata<RandomAsciiMetadata<10>, 1>)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
-                   Server_AddInitialMetadata<RandomAsciiMetadata<31>, 1>)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
-                   Server_AddInitialMetadata<RandomAsciiMetadata<100>, 1>)
-    ->Args({0, 0});
-BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
-                   Server_AddInitialMetadata<RandomAsciiMetadata<10>, 100>)
-    ->Args({0, 0});
-
-BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2)
-    ->Range(0, 128 * 1024 * 1024);
-
-static void TrickleArgs(benchmark::internal::Benchmark* b) {
-  for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) {
-    for (int j = 1; j <= 128 * 1024 * 1024; j *= 8) {
-      double expected_time =
-          static_cast<double>(14 + i) / (125.0 * static_cast<double>(j));
-      if (expected_time > 0.01) continue;
-      b->Args({i, j});
-    }
-  }
-}
-
-BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs);
-
-// Generate Args for StreamingPingPong benchmarks. Currently generates args for
-// only "small streams" (i.e streams with 0, 1 or 2 messages)
-static void StreamingPingPongArgs(benchmark::internal::Benchmark* b) {
-  int msg_size = 0;
-
-  b->Args({0, 0});  // spl case: 0 ping-pong msgs (msg_size doesn't matter here)
-
-  for (msg_size = 0; msg_size <= 128 * 1024 * 1024;
-       msg_size == 0 ? msg_size++ : msg_size *= 8) {
-    b->Args({msg_size, 1});
-    b->Args({msg_size, 2});
-  }
-}
-
-BENCHMARK_TEMPLATE(BM_StreamingPingPong, InProcessCHTTP2, NoOpMutator,
-                   NoOpMutator)
-    ->Apply(StreamingPingPongArgs);
-BENCHMARK_TEMPLATE(BM_StreamingPingPong, TCP, NoOpMutator, NoOpMutator)
-    ->Apply(StreamingPingPongArgs);
-
-BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, InProcessCHTTP2, NoOpMutator,
-                   NoOpMutator)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, TCP, NoOpMutator, NoOpMutator)
-    ->Range(0, 128 * 1024 * 1024);
-
-}  // namespace testing
-}  // namespace grpc
-
-BENCHMARK_MAIN();
diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc
new file mode 100644
index 0000000000..dc0e7d769a
--- /dev/null
+++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc
@@ -0,0 +1,197 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* Benchmark gRPC end2end in various configurations */
+
+#include <sstream>
+
+#include "src/core/lib/profiling/timers.h"
+#include "src/cpp/client/create_channel_internal.h"
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
+#include "test/cpp/microbenchmarks/fullstack_context_mutators.h"
+#include "test/cpp/microbenchmarks/fullstack_fixtures.h"
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+
+namespace grpc {
+namespace testing {
+
+// force library initialization
+auto& force_library_initialization = Library::get();
+
+/*******************************************************************************
+ * BENCHMARKING KERNELS
+ */
+
+static void* tag(intptr_t x) { return reinterpret_cast<void*>(x); }
+
+template <class Fixture>
+static void BM_PumpStreamClientToServer(benchmark::State& state) {
+  EchoTestService::AsyncService service;
+  std::unique_ptr<Fixture> fixture(new Fixture(&service));
+  {
+    EchoRequest send_request;
+    EchoRequest recv_request;
+    if (state.range(0) > 0) {
+      send_request.set_message(std::string(state.range(0), 'a'));
+    }
+    Status recv_status;
+    ServerContext svr_ctx;
+    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
+    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
+                              fixture->cq(), tag(0));
+    std::unique_ptr<EchoTestService::Stub> stub(
+        EchoTestService::NewStub(fixture->channel()));
+    ClientContext cli_ctx;
+    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
+    int need_tags = (1 << 0) | (1 << 1);
+    void* t;
+    bool ok;
+    while (need_tags) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      GPR_ASSERT(ok);
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+    response_rw.Read(&recv_request, tag(0));
+    while (state.KeepRunning()) {
+      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
+      request_rw->Write(send_request, tag(1));
+      while (true) {
+        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+        if (t == tag(0)) {
+          response_rw.Read(&recv_request, tag(0));
+        } else if (t == tag(1)) {
+          break;
+        } else {
+          GPR_ASSERT(false);
+        }
+      }
+    }
+    request_rw->WritesDone(tag(1));
+    need_tags = (1 << 0) | (1 << 1);
+    while (need_tags) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+  }
+  fixture->Finish(state);
+  fixture.reset();
+  state.SetBytesProcessed(state.range(0) * state.iterations());
+}
+
+template <class Fixture>
+static void BM_PumpStreamServerToClient(benchmark::State& state) {
+  EchoTestService::AsyncService service;
+  std::unique_ptr<Fixture> fixture(new Fixture(&service));
+  {
+    EchoResponse send_response;
+    EchoResponse recv_response;
+    if (state.range(0) > 0) {
+      send_response.set_message(std::string(state.range(0), 'a'));
+    }
+    Status recv_status;
+    ServerContext svr_ctx;
+    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
+    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
+                              fixture->cq(), tag(0));
+    std::unique_ptr<EchoTestService::Stub> stub(
+        EchoTestService::NewStub(fixture->channel()));
+    ClientContext cli_ctx;
+    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
+    int need_tags = (1 << 0) | (1 << 1);
+    void* t;
+    bool ok;
+    while (need_tags) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      GPR_ASSERT(ok);
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+    request_rw->Read(&recv_response, tag(0));
+    while (state.KeepRunning()) {
+      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
+      response_rw.Write(send_response, tag(1));
+      while (true) {
+        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+        if (t == tag(0)) {
+          request_rw->Read(&recv_response, tag(0));
+        } else if (t == tag(1)) {
+          break;
+        } else {
+          GPR_ASSERT(false);
+        }
+      }
+    }
+    response_rw.Finish(Status::OK, tag(1));
+    need_tags = (1 << 0) | (1 << 1);
+    while (need_tags) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+  }
+  fixture->Finish(state);
+  fixture.reset();
+  state.SetBytesProcessed(state.range(0) * state.iterations());
+}
+
+/*******************************************************************************
+ * CONFIGURATIONS
+ */
+
+BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2)
+    ->Range(0, 128 * 1024 * 1024);
+
+}  // namespace testing
+}  // namespace grpc
+
+BENCHMARK_MAIN();
diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc
new file mode 100644
index 0000000000..dc0e7d769a
--- /dev/null
+++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc
@@ -0,0 +1,197 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* Benchmark gRPC end2end in various configurations */
+
+#include <sstream>
+
+#include "src/core/lib/profiling/timers.h"
+#include "src/cpp/client/create_channel_internal.h"
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
+#include "test/cpp/microbenchmarks/fullstack_context_mutators.h"
+#include "test/cpp/microbenchmarks/fullstack_fixtures.h"
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+
+namespace grpc {
+namespace testing {
+
+// force library initialization
+auto& force_library_initialization = Library::get();
+
+/*******************************************************************************
+ * BENCHMARKING KERNELS
+ */
+
+static void* tag(intptr_t x) { return reinterpret_cast<void*>(x); }
+
+template <class Fixture>
+static void BM_PumpStreamClientToServer(benchmark::State& state) {
+  EchoTestService::AsyncService service;
+  std::unique_ptr<Fixture> fixture(new Fixture(&service));
+  {
+    EchoRequest send_request;
+    EchoRequest recv_request;
+    if (state.range(0) > 0) {
+      send_request.set_message(std::string(state.range(0), 'a'));
+    }
+    Status recv_status;
+    ServerContext svr_ctx;
+    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
+    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
+                              fixture->cq(), tag(0));
+    std::unique_ptr<EchoTestService::Stub> stub(
+        EchoTestService::NewStub(fixture->channel()));
+    ClientContext cli_ctx;
+    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
+    int need_tags = (1 << 0) | (1 << 1);
+    void* t;
+    bool ok;
+    while (need_tags) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      GPR_ASSERT(ok);
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+    response_rw.Read(&recv_request, tag(0));
+    while (state.KeepRunning()) {
+      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
+      request_rw->Write(send_request, tag(1));
+      while (true) {
+        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+        if (t == tag(0)) {
+          response_rw.Read(&recv_request, tag(0));
+        } else if (t == tag(1)) {
+          break;
+        } else {
+          GPR_ASSERT(false);
+        }
+      }
+    }
+    request_rw->WritesDone(tag(1));
+    need_tags = (1 << 0) | (1 << 1);
+    while (need_tags) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+  }
+  fixture->Finish(state);
+  fixture.reset();
+  state.SetBytesProcessed(state.range(0) * state.iterations());
+}
+
+template <class Fixture>
+static void BM_PumpStreamServerToClient(benchmark::State& state) {
+  EchoTestService::AsyncService service;
+  std::unique_ptr<Fixture> fixture(new Fixture(&service));
+  {
+    EchoResponse send_response;
+    EchoResponse recv_response;
+    if (state.range(0) > 0) {
+      send_response.set_message(std::string(state.range(0), 'a'));
+    }
+    Status recv_status;
+    ServerContext svr_ctx;
+    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
+    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
+                              fixture->cq(), tag(0));
+    std::unique_ptr<EchoTestService::Stub> stub(
+        EchoTestService::NewStub(fixture->channel()));
+    ClientContext cli_ctx;
+    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
+    int need_tags = (1 << 0) | (1 << 1);
+    void* t;
+    bool ok;
+    while (need_tags) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      GPR_ASSERT(ok);
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+    request_rw->Read(&recv_response, tag(0));
+    while (state.KeepRunning()) {
+      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
+      response_rw.Write(send_response, tag(1));
+      while (true) {
+        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+        if (t == tag(0)) {
+          request_rw->Read(&recv_response, tag(0));
+        } else if (t == tag(1)) {
+          break;
+        } else {
+          GPR_ASSERT(false);
+        }
+      }
+    }
+    response_rw.Finish(Status::OK, tag(1));
+    need_tags = (1 << 0) | (1 << 1);
+    while (need_tags) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+  }
+  fixture->Finish(state);
+  fixture.reset();
+  state.SetBytesProcessed(state.range(0) * state.iterations());
+}
+
+/*******************************************************************************
+ * CONFIGURATIONS
+ */
+
+BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair)
+    ->Range(0, 128 * 1024 * 1024);
+BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2)
+    ->Range(0, 128 * 1024 * 1024);
+
+}  // namespace testing
+}  // namespace grpc
+
+BENCHMARK_MAIN();
diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc
new file mode 100644
index 0000000000..5011f06368
--- /dev/null
+++ b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc
@@ -0,0 +1,219 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* Benchmark gRPC end2end in various configurations */
+
+#include "src/core/lib/profiling/timers.h"
+#include "src/cpp/client/create_channel_internal.h"
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
+#include "test/cpp/microbenchmarks/fullstack_context_mutators.h"
+#include "test/cpp/microbenchmarks/fullstack_fixtures.h"
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+extern "C" {
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/ext/transport/chttp2/transport/internal.h"
+#include "test/core/util/trickle_endpoint.h"
+}
+
+namespace grpc {
+namespace testing {
+
+static void* tag(intptr_t x) { return reinterpret_cast<void*>(x); }
+
+class TrickledCHTTP2 : public EndpointPairFixture {
+ public:
+  TrickledCHTTP2(Service* service, size_t megabits_per_second)
+      : EndpointPairFixture(service, MakeEndpoints(megabits_per_second)) {}
+
+  void AddToLabel(std::ostream& out, benchmark::State& state) {
+    out << " writes/iter:"
+        << ((double)stats_.num_writes / (double)state.iterations())
+        << " cli_transport_stalls/iter:"
+        << ((double)
+                client_stats_.streams_stalled_due_to_transport_flow_control /
+            (double)state.iterations())
+        << " cli_stream_stalls/iter:"
+        << ((double)client_stats_.streams_stalled_due_to_stream_flow_control /
+            (double)state.iterations())
+        << " svr_transport_stalls/iter:"
+        << ((double)
+                server_stats_.streams_stalled_due_to_transport_flow_control /
+            (double)state.iterations())
+        << " svr_stream_stalls/iter:"
+        << ((double)server_stats_.streams_stalled_due_to_stream_flow_control /
+            (double)state.iterations());
+  }
+
+  void Step() {
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    size_t client_backlog =
+        grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.client);
+    size_t server_backlog =
+        grpc_trickle_endpoint_trickle(&exec_ctx, endpoint_pair_.server);
+    grpc_exec_ctx_finish(&exec_ctx);
+
+    UpdateStats((grpc_chttp2_transport*)client_transport_, &client_stats_,
+                client_backlog);
+    UpdateStats((grpc_chttp2_transport*)server_transport_, &server_stats_,
+                server_backlog);
+  }
+
+ private:
+  grpc_passthru_endpoint_stats stats_;
+  struct Stats {
+    int streams_stalled_due_to_stream_flow_control = 0;
+    int streams_stalled_due_to_transport_flow_control = 0;
+  };
+  Stats client_stats_;
+  Stats server_stats_;
+
+  grpc_endpoint_pair MakeEndpoints(size_t kilobits) {
+    grpc_endpoint_pair p;
+    grpc_passthru_endpoint_create(&p.client, &p.server, Library::get().rq(),
+                                  &stats_);
+    double bytes_per_second = 125.0 * kilobits;
+    p.client = grpc_trickle_endpoint_create(p.client, bytes_per_second);
+    p.server = grpc_trickle_endpoint_create(p.server, bytes_per_second);
+    return p;
+  }
+
+  void UpdateStats(grpc_chttp2_transport* t, Stats* s, size_t backlog) {
+    if (backlog == 0) {
+      if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_STREAM].head != NULL) {
+        s->streams_stalled_due_to_stream_flow_control++;
+      }
+      if (t->lists[GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT].head != NULL) {
+        s->streams_stalled_due_to_transport_flow_control++;
+      }
+    }
+  }
+};
+
+// force library initialization
+auto& force_library_initialization = Library::get();
+
+static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok) {
+  while (true) {
+    switch (fixture->cq()->AsyncNext(
+        t, ok, gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
+                            gpr_time_from_micros(100, GPR_TIMESPAN)))) {
+      case CompletionQueue::TIMEOUT:
+        fixture->Step();
+        break;
+      case CompletionQueue::SHUTDOWN:
+        GPR_ASSERT(false);
+        break;
+      case CompletionQueue::GOT_EVENT:
+        return;
+    }
+  }
+}
+
+static void BM_PumpStreamServerToClient_Trickle(benchmark::State& state) {
+  EchoTestService::AsyncService service;
+  std::unique_ptr<TrickledCHTTP2> fixture(
+      new TrickledCHTTP2(&service, state.range(1)));
+  {
+    EchoResponse send_response;
+    EchoResponse recv_response;
+    if (state.range(0) > 0) {
+      send_response.set_message(std::string(state.range(0), 'a'));
+    }
+    Status recv_status;
+    ServerContext svr_ctx;
+    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
+    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
+                              fixture->cq(), tag(0));
+    std::unique_ptr<EchoTestService::Stub> stub(
+        EchoTestService::NewStub(fixture->channel()));
+    ClientContext cli_ctx;
+    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
+    int need_tags = (1 << 0) | (1 << 1);
+    void* t;
+    bool ok;
+    while (need_tags) {
+      TrickleCQNext(fixture.get(), &t, &ok);
+      GPR_ASSERT(ok);
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+    request_rw->Read(&recv_response, tag(0));
+    while (state.KeepRunning()) {
+      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
+      response_rw.Write(send_response, tag(1));
+      while (true) {
+        TrickleCQNext(fixture.get(), &t, &ok);
+        if (t == tag(0)) {
+          request_rw->Read(&recv_response, tag(0));
+        } else if (t == tag(1)) {
+          break;
+        } else {
+          GPR_ASSERT(false);
+        }
+      }
+    }
+    response_rw.Finish(Status::OK, tag(1));
+    need_tags = (1 << 0) | (1 << 1);
+    while (need_tags) {
+      TrickleCQNext(fixture.get(), &t, &ok);
+      int i = (int)(intptr_t)t;
+      GPR_ASSERT(need_tags & (1 << i));
+      need_tags &= ~(1 << i);
+    }
+  }
+  fixture->Finish(state);
+  fixture.reset();
+  state.SetBytesProcessed(state.range(0) * state.iterations());
+}
+
+/*******************************************************************************
+ * CONFIGURATIONS
+ */
+
+static void TrickleArgs(benchmark::internal::Benchmark* b) {
+  for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) {
+    for (int j = 1; j <= 128 * 1024 * 1024; j *= 8) {
+      double expected_time =
+          static_cast<double>(14 + i) / (125.0 * static_cast<double>(j));
+      if (expected_time > 0.01) continue;
+      b->Args({i, j});
+    }
+  }
+}
+
+BENCHMARK(BM_PumpStreamServerToClient_Trickle)->Apply(TrickleArgs);
+}
+}
+
+BENCHMARK_MAIN();
diff --git a/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc
new file mode 100644
index 0000000000..e51d272b10
--- /dev/null
+++ b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc
@@ -0,0 +1,205 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* Benchmark gRPC end2end in various configurations */
+
+#include <sstream>
+
+#include "src/core/lib/profiling/timers.h"
+#include "src/cpp/client/create_channel_internal.h"
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
+#include "test/cpp/microbenchmarks/fullstack_context_mutators.h"
+#include "test/cpp/microbenchmarks/fullstack_fixtures.h"
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+
+namespace grpc {
+namespace testing {
+
+// force library initialization
+auto& force_library_initialization = Library::get();
+
+/*******************************************************************************
+ * BENCHMARKING KERNELS
+ */
+
+static void* tag(intptr_t x) { return reinterpret_cast<void*>(x); }
+
+template <class Fixture, class ClientContextMutator, class ServerContextMutator>
+static void BM_UnaryPingPong(benchmark::State& state) {
+  EchoTestService::AsyncService service;
+  std::unique_ptr<Fixture> fixture(new Fixture(&service));
+  EchoRequest send_request;
+  EchoResponse send_response;
+  EchoResponse recv_response;
+  if (state.range(0) > 0) {
+    send_request.set_message(std::string(state.range(0), 'a'));
+  }
+  if (state.range(1) > 0) {
+    send_response.set_message(std::string(state.range(1), 'a'));
+  }
+  Status recv_status;
+  struct ServerEnv {
+    ServerContext ctx;
+    EchoRequest recv_request;
+    grpc::ServerAsyncResponseWriter<EchoResponse> response_writer;
+    ServerEnv() : response_writer(&ctx) {}
+  };
+  uint8_t server_env_buffer[2 * sizeof(ServerEnv)];
+  ServerEnv* server_env[2] = {
+      reinterpret_cast<ServerEnv*>(server_env_buffer),
+      reinterpret_cast<ServerEnv*>(server_env_buffer + sizeof(ServerEnv))};
+  new (server_env[0]) ServerEnv;
+  new (server_env[1]) ServerEnv;
+  service.RequestEcho(&server_env[0]->ctx, &server_env[0]->recv_request,
+                      &server_env[0]->response_writer, fixture->cq(),
+                      fixture->cq(), tag(0));
+  service.RequestEcho(&server_env[1]->ctx, &server_env[1]->recv_request,
+                      &server_env[1]->response_writer, fixture->cq(),
+                      fixture->cq(), tag(1));
+  std::unique_ptr<EchoTestService::Stub> stub(
+      EchoTestService::NewStub(fixture->channel()));
+  while (state.KeepRunning()) {
+    GPR_TIMER_SCOPE("BenchmarkCycle", 0);
+    recv_response.Clear();
+    ClientContext cli_ctx;
+    ClientContextMutator cli_ctx_mut(&cli_ctx);
+    std::unique_ptr<ClientAsyncResponseReader<EchoResponse>> response_reader(
+        stub->AsyncEcho(&cli_ctx, send_request, fixture->cq()));
+    void* t;
+    bool ok;
+    GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+    GPR_ASSERT(ok);
+    GPR_ASSERT(t == tag(0) || t == tag(1));
+    intptr_t slot = reinterpret_cast<intptr_t>(t);
+    ServerEnv* senv = server_env[slot];
+    ServerContextMutator svr_ctx_mut(&senv->ctx);
+    senv->response_writer.Finish(send_response, Status::OK, tag(3));
+    response_reader->Finish(&recv_response, &recv_status, tag(4));
+    for (int i = (1 << 3) | (1 << 4); i != 0;) {
+      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+      GPR_ASSERT(ok);
+      int tagnum = (int)reinterpret_cast<intptr_t>(t);
+      GPR_ASSERT(i & (1 << tagnum));
+      i -= 1 << tagnum;
+    }
+    GPR_ASSERT(recv_status.ok());
+
+    senv->~ServerEnv();
+    senv = new (senv) ServerEnv();
+    service.RequestEcho(&senv->ctx, &senv->recv_request, &senv->response_writer,
+                        fixture->cq(), fixture->cq(), tag(slot));
+  }
+  fixture->Finish(state);
+  fixture.reset();
+  server_env[0]->~ServerEnv();
+  server_env[1]->~ServerEnv();
+  state.SetBytesProcessed(state.range(0) * state.iterations() +
+                          state.range(1) * state.iterations());
+}
+
+/*******************************************************************************
+ * CONFIGURATIONS
+ */
+
+static void SweepSizesArgs(benchmark::internal::Benchmark* b) {
+  b->Args({0, 0});
+  for (int i = 1; i <= 128 * 1024 * 1024; i *= 8) {
+    b->Args({i, 0});
+    b->Args({0, i});
+    b->Args({i, i});
+  }
+}
+
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, TCP, NoOpMutator, NoOpMutator)
+    ->Apply(SweepSizesArgs);
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, UDS, NoOpMutator, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, SockPair, NoOpMutator, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator, NoOpMutator)
+    ->Apply(SweepSizesArgs);
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomBinaryMetadata<10>, 1>, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomBinaryMetadata<31>, 1>, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomBinaryMetadata<100>, 1>,
+                   NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomBinaryMetadata<10>, 2>, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomBinaryMetadata<31>, 2>, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomBinaryMetadata<100>, 2>,
+                   NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
+                   Server_AddInitialMetadata<RandomBinaryMetadata<10>, 1>)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
+                   Server_AddInitialMetadata<RandomBinaryMetadata<31>, 1>)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
+                   Server_AddInitialMetadata<RandomBinaryMetadata<100>, 1>)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomAsciiMetadata<10>, 1>, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomAsciiMetadata<31>, 1>, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2,
+                   Client_AddMetadata<RandomAsciiMetadata<100>, 1>, NoOpMutator)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
+                   Server_AddInitialMetadata<RandomAsciiMetadata<10>, 1>)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
+                   Server_AddInitialMetadata<RandomAsciiMetadata<31>, 1>)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
+                   Server_AddInitialMetadata<RandomAsciiMetadata<100>, 1>)
+    ->Args({0, 0});
+BENCHMARK_TEMPLATE(BM_UnaryPingPong, InProcessCHTTP2, NoOpMutator,
+                   Server_AddInitialMetadata<RandomAsciiMetadata<10>, 100>)
+    ->Args({0, 0});
+
+}  // namespace testing
+}  // namespace grpc
+
+BENCHMARK_MAIN();
diff --git a/test/cpp/microbenchmarks/bm_metadata.cc b/test/cpp/microbenchmarks/bm_metadata.cc
index f468690834..ee3dec2bce 100644
--- a/test/cpp/microbenchmarks/bm_metadata.cc
+++ b/test/cpp/microbenchmarks/bm_metadata.cc
@@ -42,25 +42,26 @@ extern "C" {
 #include "src/core/lib/transport/transport.h"
 }
 
+#include "test/cpp/microbenchmarks/helpers.h"
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
-static class InitializeStuff {
- public:
-  InitializeStuff() { grpc_init(); }
-  ~InitializeStuff() { grpc_shutdown(); }
-} initialize_stuff;
+auto& force_library_initialization = Library::get();
 
 static void BM_SliceFromStatic(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     benchmark::DoNotOptimize(grpc_slice_from_static_string("abc"));
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_SliceFromStatic);
 
 static void BM_SliceFromCopied(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     grpc_slice_unref(grpc_slice_from_copied_string("abc"));
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_SliceFromCopied);
 
@@ -78,38 +79,47 @@ static void BM_SliceFromStreamOwnedBuffer(benchmark::State& state) {
 BENCHMARK(BM_SliceFromStreamOwnedBuffer);
 
 static void BM_SliceIntern(benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice slice = grpc_slice_from_static_string("abc");
   while (state.KeepRunning()) {
     grpc_slice_unref(grpc_slice_intern(slice));
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_SliceIntern);
 
 static void BM_SliceReIntern(benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice slice = grpc_slice_intern(grpc_slice_from_static_string("abc"));
   while (state.KeepRunning()) {
     grpc_slice_unref(grpc_slice_intern(slice));
   }
   grpc_slice_unref(slice);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_SliceReIntern);
 
 static void BM_SliceInternStaticMetadata(benchmark::State& state) {
+  TrackCounters track_counters;
   while (state.KeepRunning()) {
     grpc_slice_intern(GRPC_MDSTR_GZIP);
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_SliceInternStaticMetadata);
 
 static void BM_SliceInternEqualToStaticMetadata(benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice slice = grpc_slice_from_static_string("gzip");
   while (state.KeepRunning()) {
     grpc_slice_intern(slice);
   }
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_SliceInternEqualToStaticMetadata);
 
 static void BM_MetadataFromNonInternedSlices(benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = grpc_slice_from_static_string("key");
   gpr_slice v = grpc_slice_from_static_string("value");
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -117,10 +127,12 @@ static void BM_MetadataFromNonInternedSlices(benchmark::State& state) {
     GRPC_MDELEM_UNREF(&exec_ctx, grpc_mdelem_create(&exec_ctx, k, v, NULL));
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromNonInternedSlices);
 
 static void BM_MetadataFromInternedSlices(benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key"));
   gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value"));
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -130,11 +142,13 @@ static void BM_MetadataFromInternedSlices(benchmark::State& state) {
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_slice_unref(k);
   grpc_slice_unref(v);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromInternedSlices);
 
 static void BM_MetadataFromInternedSlicesAlreadyInIndex(
     benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key"));
   gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value"));
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -146,10 +160,12 @@ static void BM_MetadataFromInternedSlicesAlreadyInIndex(
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_slice_unref(k);
   grpc_slice_unref(v);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromInternedSlicesAlreadyInIndex);
 
 static void BM_MetadataFromInternedKey(benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key"));
   gpr_slice v = grpc_slice_from_static_string("value");
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -158,11 +174,13 @@ static void BM_MetadataFromInternedKey(benchmark::State& state) {
   }
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_slice_unref(k);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromInternedKey);
 
 static void BM_MetadataFromNonInternedSlicesWithBackingStore(
     benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = grpc_slice_from_static_string("key");
   gpr_slice v = grpc_slice_from_static_string("value");
   char backing_store[sizeof(grpc_mdelem_data)];
@@ -174,11 +192,13 @@ static void BM_MetadataFromNonInternedSlicesWithBackingStore(
                            reinterpret_cast<grpc_mdelem_data*>(backing_store)));
   }
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromNonInternedSlicesWithBackingStore);
 
 static void BM_MetadataFromInternedSlicesWithBackingStore(
     benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key"));
   gpr_slice v = grpc_slice_intern(grpc_slice_from_static_string("value"));
   char backing_store[sizeof(grpc_mdelem_data)];
@@ -192,11 +212,13 @@ static void BM_MetadataFromInternedSlicesWithBackingStore(
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_slice_unref(k);
   grpc_slice_unref(v);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromInternedSlicesWithBackingStore);
 
 static void BM_MetadataFromInternedKeyWithBackingStore(
     benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key"));
   gpr_slice v = grpc_slice_from_static_string("value");
   char backing_store[sizeof(grpc_mdelem_data)];
@@ -209,10 +231,12 @@ static void BM_MetadataFromInternedKeyWithBackingStore(
   }
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_slice_unref(k);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromInternedKeyWithBackingStore);
 
 static void BM_MetadataFromStaticMetadataStrings(benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = GRPC_MDSTR_STATUS;
   gpr_slice v = GRPC_MDSTR_200;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -221,11 +245,13 @@ static void BM_MetadataFromStaticMetadataStrings(benchmark::State& state) {
   }
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_slice_unref(k);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromStaticMetadataStrings);
 
 static void BM_MetadataFromStaticMetadataStringsNotIndexed(
     benchmark::State& state) {
+  TrackCounters track_counters;
   gpr_slice k = GRPC_MDSTR_STATUS;
   gpr_slice v = GRPC_MDSTR_GZIP;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -234,10 +260,12 @@ static void BM_MetadataFromStaticMetadataStringsNotIndexed(
   }
   grpc_exec_ctx_finish(&exec_ctx);
   grpc_slice_unref(k);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataFromStaticMetadataStringsNotIndexed);
 
 static void BM_MetadataRefUnrefExternal(benchmark::State& state) {
+  TrackCounters track_counters;
   char backing_store[sizeof(grpc_mdelem_data)];
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_mdelem el =
@@ -249,10 +277,12 @@ static void BM_MetadataRefUnrefExternal(benchmark::State& state) {
   }
   GRPC_MDELEM_UNREF(&exec_ctx, el);
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataRefUnrefExternal);
 
 static void BM_MetadataRefUnrefInterned(benchmark::State& state) {
+  TrackCounters track_counters;
   char backing_store[sizeof(grpc_mdelem_data)];
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   gpr_slice k = grpc_slice_intern(grpc_slice_from_static_string("key"));
@@ -266,10 +296,12 @@ static void BM_MetadataRefUnrefInterned(benchmark::State& state) {
   }
   GRPC_MDELEM_UNREF(&exec_ctx, el);
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataRefUnrefInterned);
 
 static void BM_MetadataRefUnrefAllocated(benchmark::State& state) {
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_mdelem el =
       grpc_mdelem_create(&exec_ctx, grpc_slice_from_static_string("a"),
@@ -279,10 +311,12 @@ static void BM_MetadataRefUnrefAllocated(benchmark::State& state) {
   }
   GRPC_MDELEM_UNREF(&exec_ctx, el);
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataRefUnrefAllocated);
 
 static void BM_MetadataRefUnrefStatic(benchmark::State& state) {
+  TrackCounters track_counters;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_mdelem el =
       grpc_mdelem_create(&exec_ctx, GRPC_MDSTR_STATUS, GRPC_MDSTR_200, NULL);
@@ -291,6 +325,7 @@ static void BM_MetadataRefUnrefStatic(benchmark::State& state) {
   }
   GRPC_MDELEM_UNREF(&exec_ctx, el);
   grpc_exec_ctx_finish(&exec_ctx);
+  track_counters.Finish(state);
 }
 BENCHMARK(BM_MetadataRefUnrefStatic);
 
diff --git a/test/cpp/microbenchmarks/fullstack_context_mutators.h b/test/cpp/microbenchmarks/fullstack_context_mutators.h
new file mode 100644
index 0000000000..676f9aa1cc
--- /dev/null
+++ b/test/cpp/microbenchmarks/fullstack_context_mutators.h
@@ -0,0 +1,158 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef TEST_CPP_MICROBENCHMARKS_FULLSTACK_CONTEXT_MUTATORS_H
+#define TEST_CPP_MICROBENCHMARKS_FULLSTACK_CONTEXT_MUTATORS_H
+
+#include <grpc++/channel.h>
+#include <grpc++/create_channel.h>
+#include <grpc++/security/credentials.h>
+#include <grpc++/security/server_credentials.h>
+#include <grpc++/server.h>
+#include <grpc++/server_builder.h>
+#include <grpc++/server_context.h>
+#include <grpc/support/log.h>
+
+#include "test/cpp/microbenchmarks/helpers.h"
+
+namespace grpc {
+namespace testing {
+
+/*******************************************************************************
+ * CONTEXT MUTATORS
+ */
+
+static const int kPregenerateKeyCount = 100000;
+
+template <class F>
+auto MakeVector(size_t length, F f) -> std::vector<decltype(f())> {
+  std::vector<decltype(f())> out;
+  out.reserve(length);
+  for (size_t i = 0; i < length; i++) {
+    out.push_back(f());
+  }
+  return out;
+}
+
+class NoOpMutator {
+ public:
+  template <class ContextType>
+  NoOpMutator(ContextType* context) {}
+};
+
+template <int length>
+class RandomBinaryMetadata {
+ public:
+  static const grpc::string& Key() { return kKey; }
+
+  static const grpc::string& Value() {
+    return kValues[rand() % kValues.size()];
+  }
+
+ private:
+  static const grpc::string kKey;
+  static const std::vector<grpc::string> kValues;
+
+  static grpc::string GenerateOneString() {
+    grpc::string s;
+    s.reserve(length + 1);
+    for (int i = 0; i < length; i++) {
+      s += (char)rand();
+    }
+    return s;
+  }
+};
+
+template <int length>
+class RandomAsciiMetadata {
+ public:
+  static const grpc::string& Key() { return kKey; }
+
+  static const grpc::string& Value() {
+    return kValues[rand() % kValues.size()];
+  }
+
+ private:
+  static const grpc::string kKey;
+  static const std::vector<grpc::string> kValues;
+
+  static grpc::string GenerateOneString() {
+    grpc::string s;
+    s.reserve(length + 1);
+    for (int i = 0; i < length; i++) {
+      s += (char)(rand() % 26 + 'a');
+    }
+    return s;
+  }
+};
+
+template <class Generator, int kNumKeys>
+class Client_AddMetadata : public NoOpMutator {
+ public:
+  Client_AddMetadata(ClientContext* context) : NoOpMutator(context) {
+    for (int i = 0; i < kNumKeys; i++) {
+      context->AddMetadata(Generator::Key(), Generator::Value());
+    }
+  }
+};
+
+template <class Generator, int kNumKeys>
+class Server_AddInitialMetadata : public NoOpMutator {
+ public:
+  Server_AddInitialMetadata(ServerContext* context) : NoOpMutator(context) {
+    for (int i = 0; i < kNumKeys; i++) {
+      context->AddInitialMetadata(Generator::Key(), Generator::Value());
+    }
+  }
+};
+
+// static initialization
+
+template <int length>
+const grpc::string RandomBinaryMetadata<length>::kKey = "foo-bin";
+
+template <int length>
+const std::vector<grpc::string> RandomBinaryMetadata<length>::kValues =
+    MakeVector(kPregenerateKeyCount, GenerateOneString);
+
+template <int length>
+const grpc::string RandomAsciiMetadata<length>::kKey = "foo";
+
+template <int length>
+const std::vector<grpc::string> RandomAsciiMetadata<length>::kValues =
+    MakeVector(kPregenerateKeyCount, GenerateOneString);
+
+}  // namespace testing
+}  // namespace grpc
+
+#endif
diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h
new file mode 100644
index 0000000000..dc29701059
--- /dev/null
+++ b/test/cpp/microbenchmarks/fullstack_fixtures.h
@@ -0,0 +1,244 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef TEST_CPP_MICROBENCHMARKS_FULLSTACK_FIXTURES_H
+#define TEST_CPP_MICROBENCHMARKS_FULLSTACK_FIXTURES_H
+
+#include <grpc++/channel.h>
+#include <grpc++/create_channel.h>
+#include <grpc++/security/credentials.h>
+#include <grpc++/security/server_credentials.h>
+#include <grpc++/server.h>
+#include <grpc++/server_builder.h>
+#include <grpc/support/log.h>
+
+extern "C" {
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/endpoint_pair.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/tcp_posix.h"
+#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/surface/completion_queue.h"
+#include "src/core/lib/surface/server.h"
+#include "test/core/util/passthru_endpoint.h"
+#include "test/core/util/port.h"
+}
+
+#include "test/cpp/microbenchmarks/helpers.h"
+
+namespace grpc {
+namespace testing {
+
+static void ApplyCommonServerBuilderConfig(ServerBuilder* b) {
+  b->SetMaxReceiveMessageSize(INT_MAX);
+  b->SetMaxSendMessageSize(INT_MAX);
+}
+
+static void ApplyCommonChannelArguments(ChannelArguments* c) {
+  c->SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, INT_MAX);
+  c->SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, INT_MAX);
+}
+
+class BaseFixture : public TrackCounters {};
+
+class FullstackFixture : public BaseFixture {
+ public:
+  FullstackFixture(Service* service, const grpc::string& address) {
+    ServerBuilder b;
+    b.AddListeningPort(address, InsecureServerCredentials());
+    cq_ = b.AddCompletionQueue(true);
+    b.RegisterService(service);
+    ApplyCommonServerBuilderConfig(&b);
+    server_ = b.BuildAndStart();
+    ChannelArguments args;
+    ApplyCommonChannelArguments(&args);
+    channel_ = CreateCustomChannel(address, InsecureChannelCredentials(), args);
+  }
+
+  virtual ~FullstackFixture() {
+    server_->Shutdown();
+    cq_->Shutdown();
+    void* tag;
+    bool ok;
+    while (cq_->Next(&tag, &ok)) {
+    }
+  }
+
+  ServerCompletionQueue* cq() { return cq_.get(); }
+  std::shared_ptr<Channel> channel() { return channel_; }
+
+ private:
+  std::unique_ptr<Server> server_;
+  std::unique_ptr<ServerCompletionQueue> cq_;
+  std::shared_ptr<Channel> channel_;
+};
+
+class TCP : public FullstackFixture {
+ public:
+  TCP(Service* service) : FullstackFixture(service, MakeAddress()) {}
+
+ private:
+  static grpc::string MakeAddress() {
+    int port = grpc_pick_unused_port_or_die();
+    std::stringstream addr;
+    addr << "localhost:" << port;
+    return addr.str();
+  }
+};
+
+class UDS : public FullstackFixture {
+ public:
+  UDS(Service* service) : FullstackFixture(service, MakeAddress()) {}
+
+ private:
+  static grpc::string MakeAddress() {
+    int port = grpc_pick_unused_port_or_die();  // just for a unique id - not a
+                                                // real port
+    std::stringstream addr;
+    addr << "unix:/tmp/bm_fullstack." << port;
+    return addr.str();
+  }
+};
+
+class EndpointPairFixture : public BaseFixture {
+ public:
+  EndpointPairFixture(Service* service, grpc_endpoint_pair endpoints)
+      : endpoint_pair_(endpoints) {
+    ServerBuilder b;
+    cq_ = b.AddCompletionQueue(true);
+    b.RegisterService(service);
+    ApplyCommonServerBuilderConfig(&b);
+    server_ = b.BuildAndStart();
+
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+
+    /* add server endpoint to server_
+     * */
+    {
+      const grpc_channel_args* server_args =
+          grpc_server_get_channel_args(server_->c_server());
+      server_transport_ = grpc_create_chttp2_transport(
+          &exec_ctx, server_args, endpoints.server, 0 /* is_client */);
+
+      grpc_pollset** pollsets;
+      size_t num_pollsets = 0;
+      grpc_server_get_pollsets(server_->c_server(), &pollsets, &num_pollsets);
+
+      for (size_t i = 0; i < num_pollsets; i++) {
+        grpc_endpoint_add_to_pollset(&exec_ctx, endpoints.server, pollsets[i]);
+      }
+
+      grpc_server_setup_transport(&exec_ctx, server_->c_server(),
+                                  server_transport_, NULL, server_args);
+      grpc_chttp2_transport_start_reading(&exec_ctx, server_transport_, NULL);
+    }
+
+    /* create channel */
+    {
+      ChannelArguments args;
+      args.SetString(GRPC_ARG_DEFAULT_AUTHORITY, "test.authority");
+      ApplyCommonChannelArguments(&args);
+
+      grpc_channel_args c_args = args.c_channel_args();
+      client_transport_ =
+          grpc_create_chttp2_transport(&exec_ctx, &c_args, endpoints.client, 1);
+      GPR_ASSERT(client_transport_);
+      grpc_channel* channel =
+          grpc_channel_create(&exec_ctx, "target", &c_args,
+                              GRPC_CLIENT_DIRECT_CHANNEL, client_transport_);
+      grpc_chttp2_transport_start_reading(&exec_ctx, client_transport_, NULL);
+
+      channel_ = CreateChannelInternal("", channel);
+    }
+
+    grpc_exec_ctx_finish(&exec_ctx);
+  }
+
+  virtual ~EndpointPairFixture() {
+    server_->Shutdown();
+    cq_->Shutdown();
+    void* tag;
+    bool ok;
+    while (cq_->Next(&tag, &ok)) {
+    }
+  }
+
+  ServerCompletionQueue* cq() { return cq_.get(); }
+  std::shared_ptr<Channel> channel() { return channel_; }
+
+ protected:
+  grpc_endpoint_pair endpoint_pair_;
+  grpc_transport* client_transport_;
+  grpc_transport* server_transport_;
+
+ private:
+  std::unique_ptr<Server> server_;
+  std::unique_ptr<ServerCompletionQueue> cq_;
+  std::shared_ptr<Channel> channel_;
+};
+
+class SockPair : public EndpointPairFixture {
+ public:
+  SockPair(Service* service)
+      : EndpointPairFixture(service, grpc_iomgr_create_endpoint_pair(
+                                         "test", Library::get().rq(), 8192)) {}
+};
+
+class InProcessCHTTP2 : public EndpointPairFixture {
+ public:
+  InProcessCHTTP2(Service* service)
+      : EndpointPairFixture(service, MakeEndpoints()) {}
+
+  void AddToLabel(std::ostream& out, benchmark::State& state) {
+    EndpointPairFixture::AddToLabel(out, state);
+    out << " writes/iter:"
+        << ((double)stats_.num_writes / (double)state.iterations());
+  }
+
+ private:
+  grpc_passthru_endpoint_stats stats_;
+
+  grpc_endpoint_pair MakeEndpoints() {
+    grpc_endpoint_pair p;
+    grpc_passthru_endpoint_create(&p.client, &p.server, Library::get().rq(),
+                                  &stats_);
+    return p;
+  }
+};
+
+}  // namespace testing
+}  // namespace grpc
+
+#endif
diff --git a/test/cpp/microbenchmarks/helpers.cc b/test/cpp/microbenchmarks/helpers.cc
new file mode 100644
index 0000000000..947e81ffd8
--- /dev/null
+++ b/test/cpp/microbenchmarks/helpers.cc
@@ -0,0 +1,65 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "test/cpp/microbenchmarks/helpers.h"
+
+void TrackCounters::Finish(benchmark::State &state) {
+  std::ostringstream out;
+  AddToLabel(out, state);
+  auto label = out.str();
+  if (label.length() && label[0] == ' ') {
+    label = label.substr(1);
+  }
+  state.SetLabel(label);
+}
+
+void TrackCounters::AddToLabel(std::ostream &out, benchmark::State &state) {
+#ifdef GPR_LOW_LEVEL_COUNTERS
+  out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) -
+                                     mu_locks_at_start_) /
+                            (double)state.iterations())
+      << " atm_cas/iter:"
+      << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_cas) -
+                   atm_cas_at_start_) /
+          (double)state.iterations())
+      << " atm_add/iter:"
+      << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) -
+                   atm_add_at_start_) /
+          (double)state.iterations());
+#endif
+  grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot();
+  out << " allocs/iter:"
+      << ((double)(counters_at_end.total_allocs_absolute -
+                   counters_at_start_.total_allocs_absolute) /
+          (double)state.iterations());
+}
diff --git a/test/cpp/microbenchmarks/helpers.h b/test/cpp/microbenchmarks/helpers.h
new file mode 100644
index 0000000000..42a8fbaf0b
--- /dev/null
+++ b/test/cpp/microbenchmarks/helpers.h
@@ -0,0 +1,91 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef TEST_CPP_MICROBENCHMARKS_COUNTERS_H
+#define TEST_CPP_MICROBENCHMARKS_COUNTERS_H
+
+#include <sstream>
+
+extern "C" {
+#include <grpc/support/port_platform.h>
+#include "test/core/util/memory_counters.h"
+}
+
+#include <grpc++/impl/grpc_library.h>
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+
+class Library {
+ public:
+  static Library& get() {
+    static Library lib;
+    return lib;
+  }
+
+  grpc_resource_quota* rq() { return rq_; }
+
+ private:
+  Library() {
+    grpc_memory_counters_init();
+    init_lib_.init();
+    rq_ = grpc_resource_quota_create("bm");
+  }
+
+  ~Library() { init_lib_.shutdown(); }
+
+  grpc::internal::GrpcLibrary init_lib_;
+  grpc_resource_quota* rq_;
+};
+
+#ifdef GPR_LOW_LEVEL_COUNTERS
+extern "C" gpr_atm gpr_mu_locks;
+extern "C" gpr_atm gpr_counter_atm_cas;
+extern "C" gpr_atm gpr_counter_atm_add;
+#endif
+
+class TrackCounters {
+ public:
+  virtual void Finish(benchmark::State& state);
+  virtual void AddToLabel(std::ostream& out, benchmark::State& state);
+
+ private:
+#ifdef GPR_LOW_LEVEL_COUNTERS
+  const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks);
+  const size_t atm_cas_at_start_ =
+      gpr_atm_no_barrier_load(&gpr_counter_atm_cas);
+  const size_t atm_add_at_start_ =
+      gpr_atm_no_barrier_load(&gpr_counter_atm_add);
+#endif
+  grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot();
+};
+
+#endif