Merge branch 'master' into cq_create_api_changes

48 files changed, 1269 insertions, 181 deletions

diff --git a/test/core/channel/channel_stack_test.c b/test/core/channel/channel_stack_test.c
index 76bb57346c..af551c4928 100644
--- a/test/core/channel/channel_stack_test.c
+++ b/test/core/channel/channel_stack_test.c
@@ -68,7 +68,7 @@ static void channel_destroy_func(grpc_exec_ctx *exec_ctx,
 
 static void call_destroy_func(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                               const grpc_call_final_info *final_info,
-                              void *ignored) {
+                              grpc_closure *ignored) {
   ++*(int *)(elem->channel_data);
 }
 
@@ -139,10 +139,16 @@ static void test_create_channel_stack(void) {
   GPR_ASSERT(*channel_data == 0);
 
   call_stack = gpr_malloc(channel_stack->call_stack_size);
-  grpc_error *error =
-      grpc_call_stack_init(&exec_ctx, channel_stack, 1, free_call, call_stack,
-                           NULL, NULL, path, gpr_now(GPR_CLOCK_MONOTONIC),
-                           gpr_inf_future(GPR_CLOCK_MONOTONIC), call_stack);
+  const grpc_call_element_args args = {
+      .call_stack = call_stack,
+      .server_transport_data = NULL,
+      .context = NULL,
+      .path = path,
+      .start_time = gpr_now(GPR_CLOCK_MONOTONIC),
+      .deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC),
+      .arena = NULL};
+  grpc_error *error = grpc_call_stack_init(&exec_ctx, channel_stack, 1,
+                                           free_call, call_stack, &args);
   GPR_ASSERT(error == GRPC_ERROR_NONE);
   GPR_ASSERT(call_stack->count == 1);
   call_elem = grpc_call_stack_element(call_stack, 0);
diff --git a/test/core/end2end/end2end_tests.h b/test/core/end2end/end2end_tests.h
index ac87c55dc4..4d98bddbd8 100644
--- a/test/core/end2end/end2end_tests.h
+++ b/test/core/end2end/end2end_tests.h
@@ -39,12 +39,15 @@
 typedef struct grpc_end2end_test_fixture grpc_end2end_test_fixture;
 typedef struct grpc_end2end_test_config grpc_end2end_test_config;
 
+/* Test feature flags. */
 #define FEATURE_MASK_SUPPORTS_DELAYED_CONNECTION 1
 #define FEATURE_MASK_SUPPORTS_HOSTNAME_VERIFICATION 2
 #define FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS 4
 #define FEATURE_MASK_SUPPORTS_REQUEST_PROXYING 8
 #define FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL 16
 #define FEATURE_MASK_SUPPORTS_AUTHORITY_HEADER 32
+#define FEATURE_MASK_DOES_NOT_SUPPORT_RESOURCE_QUOTA_SERVER 64
+#define FEATURE_MASK_DOES_NOT_SUPPORT_NETWORK_STATUS_CHANGE 128
 
 #define FAIL_AUTH_CHECK_SERVER_ARG_NAME "fail_auth_check"
 
@@ -57,8 +60,12 @@ struct grpc_end2end_test_fixture {
 };
 
 struct grpc_end2end_test_config {
+  /* A descriptive name for this test fixture. */
   const char *name;
+
+  /* Which features are supported by this fixture. See feature flags above. */
   uint32_t feature_mask;
+
   grpc_end2end_test_fixture (*create_fixture)(grpc_channel_args *client_args,
                                               grpc_channel_args *server_args);
   void (*init_client)(grpc_end2end_test_fixture *f,
diff --git a/test/core/end2end/tests/filter_call_init_fails.c b/test/core/end2end/tests/filter_call_init_fails.c
index dee3531f80..ed193ea042 100644
--- a/test/core/end2end/tests/filter_call_init_fails.c
+++ b/test/core/end2end/tests/filter_call_init_fails.c
@@ -215,7 +215,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
 
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                               const grpc_call_final_info *final_info,
-                              void *and_free_memory) {}
+                              grpc_closure *ignored) {}
 
 static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx,
                                      grpc_channel_element *elem,
diff --git a/test/core/end2end/tests/filter_causes_close.c b/test/core/end2end/tests/filter_causes_close.c
index d41eb48101..29acd5b131 100644
--- a/test/core/end2end/tests/filter_causes_close.c
+++ b/test/core/end2end/tests/filter_causes_close.c
@@ -238,7 +238,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
 
 static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                               const grpc_call_final_info *final_info,
-                              void *and_free_memory) {}
+                              grpc_closure *ignored) {}
 
 static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx,
                                      grpc_channel_element *elem,
diff --git a/test/core/end2end/tests/filter_latency.c b/test/core/end2end/tests/filter_latency.c
index ffe9880d81..9f380670f8 100644
--- a/test/core/end2end/tests/filter_latency.c
+++ b/test/core/end2end/tests/filter_latency.c
@@ -269,7 +269,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
 static void client_destroy_call_elem(grpc_exec_ctx *exec_ctx,
                                      grpc_call_element *elem,
                                      const grpc_call_final_info *final_info,
-                                     void *and_free_memory) {
+                                     grpc_closure *ignored) {
   gpr_mu_lock(&g_mu);
   g_client_latency = final_info->stats.latency;
   gpr_mu_unlock(&g_mu);
@@ -278,7 +278,7 @@ static void client_destroy_call_elem(grpc_exec_ctx *exec_ctx,
 static void server_destroy_call_elem(grpc_exec_ctx *exec_ctx,
                                      grpc_call_element *elem,
                                      const grpc_call_final_info *final_info,
-                                     void *and_free_memory) {
+                                     grpc_closure *ignored) {
   gpr_mu_lock(&g_mu);
   g_server_latency = final_info->stats.latency;
   gpr_mu_unlock(&g_mu);
diff --git a/test/core/end2end/tests/network_status_change.c b/test/core/end2end/tests/network_status_change.c
index 3a420cce23..9d37f83823 100644
--- a/test/core/end2end/tests/network_status_change.c
+++ b/test/core/end2end/tests/network_status_change.c
@@ -242,6 +242,10 @@ static void test_invoke_network_status_change(grpc_end2end_test_config config) {
 }
 
 void network_status_change(grpc_end2end_test_config config) {
+  if (config.feature_mask &
+      FEATURE_MASK_DOES_NOT_SUPPORT_NETWORK_STATUS_CHANGE) {
+    return;
+  }
   test_invoke_network_status_change(config);
 }
 
diff --git a/test/core/end2end/tests/resource_quota_server.c b/test/core/end2end/tests/resource_quota_server.c
index 115fddc167..1e31f03879 100644
--- a/test/core/end2end/tests/resource_quota_server.c
+++ b/test/core/end2end/tests/resource_quota_server.c
@@ -115,6 +115,10 @@ static grpc_slice generate_random_slice() {
 }
 
 void resource_quota_server(grpc_end2end_test_config config) {
+  if (config.feature_mask &
+      FEATURE_MASK_DOES_NOT_SUPPORT_RESOURCE_QUOTA_SERVER) {
+    return;
+  }
   grpc_resource_quota *resource_quota =
       grpc_resource_quota_create("test_server");
   grpc_resource_quota_resize(resource_quota, 5 * 1024 * 1024);
diff --git a/test/core/end2end/tests/streaming_error_response.c b/test/core/end2end/tests/streaming_error_response.c
index 354c5091fd..51cb78df20 100644
--- a/test/core/end2end/tests/streaming_error_response.c
+++ b/test/core/end2end/tests/streaming_error_response.c
@@ -31,6 +31,9 @@
  *
  */
 
+/** \file Verify that status ordering rules are obeyed.
+    \ref doc/status_ordering.md */
+
 #include "test/core/end2end/end2end_tests.h"
 
 #include <stdio.h>
diff --git a/test/core/iomgr/error_test.c b/test/core/iomgr/error_test.c
new file mode 100644
index 0000000000..2a6b1b17fd
--- /dev/null
+++ b/test/core/iomgr/error_test.c
@@ -0,0 +1,219 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/iomgr/error.h"
+
+#include <grpc/grpc.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/thd.h>
+#include <grpc/support/useful.h>
+
+#include <string.h>
+
+#include "test/core/util/test_config.h"
+
+static void test_set_get_int() {
+  grpc_error* error = GRPC_ERROR_CREATE("Test");
+  GPR_ASSERT(error);
+  intptr_t i = 0;
+  GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_FILE_LINE, &i));
+  GPR_ASSERT(i);  // line set will never be 0
+  GPR_ASSERT(!grpc_error_get_int(error, GRPC_ERROR_INT_ERRNO, &i));
+  GPR_ASSERT(!grpc_error_get_int(error, GRPC_ERROR_INT_SIZE, &i));
+
+  intptr_t errnumber = 314;
+  error = grpc_error_set_int(error, GRPC_ERROR_INT_ERRNO, errnumber);
+  GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_ERRNO, &i));
+  GPR_ASSERT(i == errnumber);
+
+  intptr_t http = 2;
+  error = grpc_error_set_int(error, GRPC_ERROR_INT_HTTP2_ERROR, http);
+  GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_HTTP2_ERROR, &i));
+  GPR_ASSERT(i == http);
+
+  GRPC_ERROR_UNREF(error);
+}
+
+static void test_set_get_str() {
+  grpc_error* error = GRPC_ERROR_CREATE("Test");
+
+  GPR_ASSERT(!grpc_error_get_str(error, GRPC_ERROR_STR_SYSCALL));
+  GPR_ASSERT(!grpc_error_get_str(error, GRPC_ERROR_STR_TSI_ERROR));
+
+  const char* c = grpc_error_get_str(error, GRPC_ERROR_STR_FILE);
+  GPR_ASSERT(c);
+  GPR_ASSERT(strstr(c, "error_test.c"));  // __FILE__ expands differently on
+                                          // Windows. All should at least
+                                          // contain error_test.c
+
+  c = grpc_error_get_str(error, GRPC_ERROR_STR_DESCRIPTION);
+  GPR_ASSERT(c);
+  GPR_ASSERT(!strcmp(c, "Test"));
+
+  error =
+      grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, "longer message");
+  c = grpc_error_get_str(error, GRPC_ERROR_STR_GRPC_MESSAGE);
+  GPR_ASSERT(c);
+  GPR_ASSERT(!strcmp(c, "longer message"));
+
+  GRPC_ERROR_UNREF(error);
+}
+
+static void test_copy_and_unref() {
+  // error1 has one ref
+  grpc_error* error1 = grpc_error_set_str(
+      GRPC_ERROR_CREATE("Test"), GRPC_ERROR_STR_GRPC_MESSAGE, "message");
+  const char* c = grpc_error_get_str(error1, GRPC_ERROR_STR_GRPC_MESSAGE);
+  GPR_ASSERT(c);
+  GPR_ASSERT(!strcmp(c, "message"));
+
+  // error 1 has two refs
+  GRPC_ERROR_REF(error1);
+  // this gives error3 a ref to the new error, and decrements error1 to one ref
+  grpc_error* error3 =
+      grpc_error_set_str(error1, GRPC_ERROR_STR_SYSCALL, "syscall");
+  GPR_ASSERT(error3 != error1);  // should not be the same because of extra ref
+  c = grpc_error_get_str(error3, GRPC_ERROR_STR_GRPC_MESSAGE);
+  GPR_ASSERT(c);
+  GPR_ASSERT(!strcmp(c, "message"));
+
+  // error 1 should not have a syscall but 3 should
+  GPR_ASSERT(!grpc_error_get_str(error1, GRPC_ERROR_STR_SYSCALL));
+  c = grpc_error_get_str(error3, GRPC_ERROR_STR_SYSCALL);
+  GPR_ASSERT(c);
+  GPR_ASSERT(!strcmp(c, "syscall"));
+
+  GRPC_ERROR_UNREF(error1);
+  GRPC_ERROR_UNREF(error3);
+}
+
+static void test_create_referencing() {
+  grpc_error* child = grpc_error_set_str(
+      GRPC_ERROR_CREATE("Child"), GRPC_ERROR_STR_GRPC_MESSAGE, "message");
+  grpc_error* parent = GRPC_ERROR_CREATE_REFERENCING("Parent", &child, 1);
+  GPR_ASSERT(parent);
+
+  GRPC_ERROR_UNREF(child);
+  GRPC_ERROR_UNREF(parent);
+}
+
+static void test_create_referencing_many() {
+  grpc_error* children[3];
+  children[0] = grpc_error_set_str(GRPC_ERROR_CREATE("Child1"),
+                                   GRPC_ERROR_STR_GRPC_MESSAGE, "message");
+  children[1] = grpc_error_set_int(GRPC_ERROR_CREATE("Child2"),
+                                   GRPC_ERROR_INT_HTTP2_ERROR, 5);
+  children[2] = grpc_error_set_str(GRPC_ERROR_CREATE("Child3"),
+                                   GRPC_ERROR_STR_GRPC_MESSAGE, "message 3");
+
+  grpc_error* parent = GRPC_ERROR_CREATE_REFERENCING("Parent", children, 3);
+  GPR_ASSERT(parent);
+
+  for (size_t i = 0; i < 3; ++i) {
+    GRPC_ERROR_UNREF(children[i]);
+  }
+  GRPC_ERROR_UNREF(parent);
+}
+
+static void print_error_string() {
+  grpc_error* error =
+      grpc_error_set_int(GRPC_ERROR_CREATE("Error"), GRPC_ERROR_INT_GRPC_STATUS,
+                         GRPC_STATUS_UNIMPLEMENTED);
+  error = grpc_error_set_int(error, GRPC_ERROR_INT_SIZE, 666);
+  error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, "message");
+  // gpr_log(GPR_DEBUG, "%s", grpc_error_string(error));
+  GRPC_ERROR_UNREF(error);
+}
+
+static void print_error_string_reference() {
+  grpc_error* children[2];
+  children[0] = grpc_error_set_str(
+      grpc_error_set_int(GRPC_ERROR_CREATE("1"), GRPC_ERROR_INT_GRPC_STATUS,
+                         GRPC_STATUS_UNIMPLEMENTED),
+      GRPC_ERROR_STR_GRPC_MESSAGE, "message for child 1");
+  children[1] = grpc_error_set_str(
+      grpc_error_set_int(GRPC_ERROR_CREATE("2sd"), GRPC_ERROR_INT_GRPC_STATUS,
+                         GRPC_STATUS_INTERNAL),
+      GRPC_ERROR_STR_GRPC_MESSAGE, "message for child 2");
+
+  grpc_error* parent = GRPC_ERROR_CREATE_REFERENCING("Parent", children, 2);
+
+  gpr_log(GPR_DEBUG, "%s", grpc_error_string(parent));
+
+  for (size_t i = 0; i < 2; ++i) {
+    GRPC_ERROR_UNREF(children[i]);
+  }
+  GRPC_ERROR_UNREF(parent);
+}
+
+static void test_os_error() {
+  int fake_errno = 5;
+  const char* syscall = "syscall name";
+  grpc_error* error = GRPC_OS_ERROR(fake_errno, syscall);
+
+  intptr_t i = 0;
+  GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_ERRNO, &i));
+  GPR_ASSERT(i == fake_errno);
+
+  const char* c = grpc_error_get_str(error, GRPC_ERROR_STR_SYSCALL);
+  GPR_ASSERT(c);
+  GPR_ASSERT(!strcmp(c, syscall));
+  GRPC_ERROR_UNREF(error);
+}
+
+static void test_special() {
+  grpc_error* error = GRPC_ERROR_NONE;
+  error = grpc_error_add_child(error, GRPC_ERROR_CREATE("test child"));
+  intptr_t i;
+  GPR_ASSERT(grpc_error_get_int(error, GRPC_ERROR_INT_GRPC_STATUS, &i));
+  GPR_ASSERT(i == GRPC_STATUS_OK);
+  GRPC_ERROR_UNREF(error);
+}
+
+int main(int argc, char** argv) {
+  grpc_test_init(argc, argv);
+  grpc_init();
+  test_set_get_int();
+  test_set_get_str();
+  test_copy_and_unref();
+  print_error_string();
+  print_error_string_reference();
+  test_os_error();
+  test_create_referencing();
+  test_create_referencing_many();
+  test_special();
+  grpc_shutdown();
+
+  return 0;
+}
diff --git a/test/core/iomgr/tcp_client_uv_test.c b/test/core/iomgr/tcp_client_uv_test.c
index f8938d0abb..064119f11b 100644
--- a/test/core/iomgr/tcp_client_uv_test.c
+++ b/test/core/iomgr/tcp_client_uv_test.c
@@ -58,7 +58,7 @@ static int g_connections_complete = 0;
 static grpc_endpoint *g_connecting = NULL;
 
 static gpr_timespec test_deadline(void) {
-  return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10);
+  return grpc_timeout_seconds_to_deadline(10);
 }
 
 static void finish_connection() {
@@ -73,7 +73,8 @@ static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg,
                          grpc_error *error) {
   GPR_ASSERT(g_connecting != NULL);
   GPR_ASSERT(error == GRPC_ERROR_NONE);
-  grpc_endpoint_shutdown(exec_ctx, g_connecting);
+  grpc_endpoint_shutdown(exec_ctx, g_connecting,
+                         GRPC_ERROR_CREATE("must_succeed called"));
   grpc_endpoint_destroy(exec_ctx, g_connecting);
   g_connecting = NULL;
   finish_connection();
@@ -133,7 +134,7 @@ void test_succeeds(void) {
         "pollset_work",
         grpc_pollset_work(&exec_ctx, g_pollset, &worker,
                           gpr_now(GPR_CLOCK_MONOTONIC),
-                          GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5))));
+                          grpc_timeout_seconds_to_deadline(5))));
     gpr_mu_unlock(g_mu);
     grpc_exec_ctx_flush(&exec_ctx);
     gpr_mu_lock(g_mu);
diff --git a/test/core/iomgr/tcp_server_uv_test.c b/test/core/iomgr/tcp_server_uv_test.c
index 7b458c90f3..0fc74599ea 100644
--- a/test/core/iomgr/tcp_server_uv_test.c
+++ b/test/core/iomgr/tcp_server_uv_test.c
@@ -115,7 +115,7 @@ static void server_weak_ref_set(server_weak_ref *weak_ref,
 static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp,
                        grpc_pollset *pollset,
                        grpc_tcp_server_acceptor *acceptor) {
-  grpc_endpoint_shutdown(exec_ctx, tcp);
+  grpc_endpoint_shutdown(exec_ctx, tcp, GRPC_ERROR_CREATE("Connected"));
   grpc_endpoint_destroy(exec_ctx, tcp);
 
   on_connect_result temp_result;
@@ -203,7 +203,7 @@ static void close_cb(uv_handle_t *handle) { gpr_free(handle); }
 
 static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote,
                         socklen_t remote_len, on_connect_result *result) {
-  gpr_timespec deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10);
+  gpr_timespec deadline = grpc_timeout_seconds_to_deadline(10);
   uv_tcp_t *client_handle = gpr_malloc(sizeof(uv_tcp_t));
   uv_connect_t *req = gpr_malloc(sizeof(uv_connect_t));
   int nconnects_before;
diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c
index d57a37b2a9..396ec959cd 100644
--- a/test/core/iomgr/udp_server_test.c
+++ b/test/core/iomgr/udp_server_test.c
@@ -62,8 +62,7 @@ static int g_number_of_writes = 0;
 static int g_number_of_bytes_read = 0;
 static int g_number_of_orphan_calls = 0;
 
-static void on_read(grpc_exec_ctx *exec_ctx, grpc_fd *emfd,
-                    grpc_server *server) {
+static void on_read(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) {
   char read_buffer[512];
   ssize_t byte_count;
 
@@ -79,7 +78,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, grpc_fd *emfd,
   gpr_mu_unlock(g_mu);
 }
 
-static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd) {
+static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) {
   gpr_mu_lock(g_mu);
   g_number_of_writes++;
 
@@ -88,7 +87,8 @@ static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd) {
   gpr_mu_unlock(g_mu);
 }
 
-static void on_fd_orphaned(grpc_exec_ctx *exec_ctx, grpc_fd *emfd) {
+static void on_fd_orphaned(grpc_exec_ctx *exec_ctx, grpc_fd *emfd,
+                           void *user_data) {
   gpr_log(GPR_INFO, "gRPC FD about to be orphaned: %d",
           grpc_fd_wrapped_fd(emfd));
   g_number_of_orphan_calls++;
diff --git a/test/core/memory_usage/client.c b/test/core/memory_usage/client.c
index d3eb5ac7c2..582ede48a8 100644
--- a/test/core/memory_usage/client.c
+++ b/test/core/memory_usage/client.c
@@ -43,6 +43,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
+#include "src/core/lib/support/env.h"
 #include "src/core/lib/support/string.h"
 #include "test/core/util/memory_counters.h"
 #include "test/core/util/test_config.h"
@@ -311,6 +312,29 @@ int main(int argc, char **argv) {
           server_calls_end.total_size_relative -
               after_server_create.total_size_relative);
 
+  const char *csv_file = "memory_usage.csv";
+  FILE *csv = fopen(csv_file, "w");
+  if (csv) {
+    char *env_build = gpr_getenv("BUILD_NUMBER");
+    char *env_job = gpr_getenv("JOB_NAME");
+    fprintf(csv, "%f,%zi,%zi,%f,%zi,%s,%s\n",
+            (double)(client_calls_inflight.total_size_relative -
+                     client_benchmark_calls_start.total_size_relative) /
+                benchmark_iterations,
+            client_channel_end.total_size_relative -
+                client_channel_start.total_size_relative,
+            after_server_create.total_size_relative -
+                before_server_create.total_size_relative,
+            (double)(server_calls_inflight.total_size_relative -
+                     server_benchmark_calls_start.total_size_relative) /
+                benchmark_iterations,
+            server_calls_end.total_size_relative -
+                after_server_create.total_size_relative,
+            env_build == NULL ? "" : env_build, env_job == NULL ? "" : env_job);
+    fclose(csv);
+    gpr_log(GPR_INFO, "Summary written to %s", csv_file);
+  }
+
   grpc_memory_counters_destroy();
   return 0;
 }
diff --git a/test/core/security/BUILD b/test/core/security/BUILD
index e750c39b7c..1cb03c5cfe 100644
--- a/test/core/security/BUILD
+++ b/test/core/security/BUILD
@@ -34,7 +34,7 @@ load("//test/core/util:grpc_fuzzer.bzl", "grpc_fuzzer")
 grpc_fuzzer(
   name = "ssl_server_fuzzer",
   srcs = ["ssl_server_fuzzer.c"],
-  deps = ["//:gpr", "//:grpc", "//test/core/util:grpc_test_util"],
+  deps = ["//:gpr", "//:grpc", "//test/core/util:grpc_test_util", "//test/core/end2end:ssl_test_data"],
   corpus = "corpus",
   copts = ["-std=c99"],
 )
diff --git a/test/core/security/ssl_server_fuzzer.c b/test/core/security/ssl_server_fuzzer.c
index f789278add..7a3612c419 100644
--- a/test/core/security/ssl_server_fuzzer.c
+++ b/test/core/security/ssl_server_fuzzer.c
@@ -38,6 +38,7 @@
 #include "src/core/lib/iomgr/load_file.h"
 #include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/security/transport/security_connector.h"
+#include "test/core/end2end/data/ssl_test_data.h"
 #include "test/core/util/memory_counters.h"
 #include "test/core/util/mock_endpoint.h"
 
@@ -46,10 +47,6 @@ bool squelch = true;
 // Turning this on will fail the leak check.
 bool leak_check = false;
 
-#define SSL_CERT_PATH "src/core/lib/tsi/test_creds/server1.pem"
-#define SSL_KEY_PATH "src/core/lib/tsi/test_creds/server1.key"
-#define SSL_CA_PATH "src/core/lib/tsi/test_creds/ca.pem"
-
 static void discard_write(grpc_slice slice) {}
 
 static void dont_log(gpr_log_func_args *args) {}
@@ -88,12 +85,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   // Load key pair and establish server SSL credentials.
   grpc_ssl_pem_key_cert_pair pem_key_cert_pair;
   grpc_slice ca_slice, cert_slice, key_slice;
-  GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file",
-                               grpc_load_file(SSL_CA_PATH, 1, &ca_slice)));
-  GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file",
-                               grpc_load_file(SSL_CERT_PATH, 1, &cert_slice)));
-  GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file",
-                               grpc_load_file(SSL_KEY_PATH, 1, &key_slice)));
+  ca_slice = grpc_slice_from_static_string(test_root_cert);
+  cert_slice = grpc_slice_from_static_string(test_server1_cert);
+  key_slice = grpc_slice_from_static_string(test_server1_key);
   const char *ca_cert = (const char *)GRPC_SLICE_START_PTR(ca_slice);
   pem_key_cert_pair.private_key = (const char *)GRPC_SLICE_START_PTR(key_slice);
   pem_key_cert_pair.cert_chain = (const char *)GRPC_SLICE_START_PTR(cert_slice);
diff --git a/test/core/support/arena_test.c b/test/core/support/arena_test.c
new file mode 100644
index 0000000000..35b2bbd1b1
--- /dev/null
+++ b/test/core/support/arena_test.c
@@ -0,0 +1,139 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/support/arena.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/thd.h>
+#include <grpc/support/useful.h>
+#include <inttypes.h>
+#include <string.h>
+
+#include "src/core/lib/support/string.h"
+#include "test/core/util/test_config.h"
+
+static void test_noop(void) { gpr_arena_destroy(gpr_arena_create(1)); }
+
+static void test(const char *name, size_t init_size, const size_t *allocs,
+                 size_t nallocs) {
+  gpr_strvec v;
+  char *s;
+  gpr_strvec_init(&v);
+  gpr_asprintf(&s, "test '%s': %" PRIdPTR " <- {", name, init_size);
+  gpr_strvec_add(&v, s);
+  for (size_t i = 0; i < nallocs; i++) {
+    gpr_asprintf(&s, "%" PRIdPTR ",", allocs[i]);
+    gpr_strvec_add(&v, s);
+  }
+  gpr_strvec_add(&v, gpr_strdup("}"));
+  s = gpr_strvec_flatten(&v, NULL);
+  gpr_strvec_destroy(&v);
+  gpr_log(GPR_INFO, "%s", s);
+  gpr_free(s);
+
+  gpr_arena *a = gpr_arena_create(init_size);
+  void **ps = gpr_zalloc(sizeof(*ps) * nallocs);
+  for (size_t i = 0; i < nallocs; i++) {
+    ps[i] = gpr_arena_alloc(a, allocs[i]);
+    // ensure no duplicate results
+    for (size_t j = 0; j < i; j++) {
+      GPR_ASSERT(ps[i] != ps[j]);
+    }
+    // ensure writable
+    memset(ps[i], 1, allocs[i]);
+  }
+  gpr_arena_destroy(a);
+  gpr_free(ps);
+}
+
+#define TEST(name, init_size, ...)                     \
+  static const size_t allocs_##name[] = {__VA_ARGS__}; \
+  test(#name, init_size, allocs_##name, GPR_ARRAY_SIZE(allocs_##name))
+
+#define CONCURRENT_TEST_ITERATIONS 100000
+#define CONCURRENT_TEST_THREADS 100
+
+typedef struct {
+  gpr_event ev_start;
+  gpr_arena *arena;
+} concurrent_test_args;
+
+static void concurrent_test_body(void *arg) {
+  concurrent_test_args *a = arg;
+  gpr_event_wait(&a->ev_start, gpr_inf_future(GPR_CLOCK_REALTIME));
+  for (size_t i = 0; i < CONCURRENT_TEST_ITERATIONS; i++) {
+    *(char *)gpr_arena_alloc(a->arena, 1) = (char)i;
+  }
+}
+
+static void concurrent_test(void) {
+  gpr_log(GPR_DEBUG, "concurrent_test");
+
+  concurrent_test_args args;
+  gpr_event_init(&args.ev_start);
+  args.arena = gpr_arena_create(1024);
+
+  gpr_thd_id thds[CONCURRENT_TEST_THREADS];
+
+  for (int i = 0; i < CONCURRENT_TEST_THREADS; i++) {
+    gpr_thd_options opt = gpr_thd_options_default();
+    gpr_thd_options_set_joinable(&opt);
+    gpr_thd_new(&thds[i], concurrent_test_body, &args, &opt);
+  }
+
+  gpr_event_set(&args.ev_start, (void *)1);
+
+  for (int i = 0; i < CONCURRENT_TEST_THREADS; i++) {
+    gpr_thd_join(thds[i]);
+  }
+
+  gpr_arena_destroy(args.arena);
+}
+
+int main(int argc, char *argv[]) {
+  grpc_test_init(argc, argv);
+
+  test_noop();
+  TEST(0_1, 0, 1);
+  TEST(1_1, 1, 1);
+  TEST(1_2, 1, 2);
+  TEST(1_3, 1, 3);
+  TEST(1_inc, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);
+  TEST(6_123, 6, 1, 2, 3);
+  concurrent_test();
+
+  return 0;
+}
diff --git a/test/core/support/cpu_test.c b/test/core/support/cpu_test.c
index ca0fe0ccb5..7b9bf6c5e1 100644
--- a/test/core/support/cpu_test.c
+++ b/test/core/support/cpu_test.c
@@ -81,9 +81,12 @@ static void worker_thread(void *arg) {
   uint32_t cpu;
   unsigned r = 12345678;
   unsigned i, j;
-  for (i = 0; i < 1000 / grpc_test_slowdown_factor(); i++) {
+  /* Avoid repetitive division calculations */
+  int64_t max_i = 1000 / grpc_test_slowdown_factor();
+  int64_t max_j = 1000000 / grpc_test_slowdown_factor();
+  for (i = 0; i < max_i; i++) {
     /* run for a bit - just calculate something random. */
-    for (j = 0; j < 1000000 / grpc_test_slowdown_factor(); j++) {
+    for (j = 0; j < max_j; j++) {
       r = (r * 17) & ((r - i) | (r * i));
     }
     cpu = gpr_cpu_current_cpu();
diff --git a/test/core/transport/chttp2/hpack_parser_corpus/clusterfuzz-testcase-5298216461402112 b/test/core/transport/chttp2/hpack_parser_corpus/clusterfuzz-testcase-5298216461402112
new file mode 100644
index 0000000000..04d48d6d76
--- /dev/null
+++ b/test/core/transport/chttp2/hpack_parser_corpus/clusterfuzz-testcase-5298216461402112
diff --git a/test/core/util/memory_counters.c b/test/core/util/memory_counters.c
index 7c8b620f34..c27065f260 100644
--- a/test/core/util/memory_counters.c
+++ b/test/core/util/memory_counters.c
@@ -46,17 +46,23 @@ static void *guard_malloc(size_t size);
 static void *guard_realloc(void *vptr, size_t size);
 static void guard_free(void *vptr);
 
+#ifdef GPR_LOW_LEVEL_COUNTERS
+/* hide these from the microbenchmark atomic stats */
+#define NO_BARRIER_FETCH_ADD(x, sz) \
+  __atomic_fetch_add((x), (sz), __ATOMIC_RELAXED)
+#define NO_BARRIER_LOAD(x) __atomic_load_n((x), __ATOMIC_RELAXED)
+#else
+#define NO_BARRIER_FETCH_ADD(x, sz) gpr_atm_no_barrier_fetch_add(x, sz)
+#define NO_BARRIER_LOAD(x) gpr_atm_no_barrier_load(x)
+#endif
+
 static void *guard_malloc(size_t size) {
   size_t *ptr;
   if (!size) return NULL;
-  gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_absolute,
-                               (gpr_atm)size);
-  gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative,
-                               (gpr_atm)size);
-  gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_absolute,
-                               (gpr_atm)1);
-  gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_relative,
-                               (gpr_atm)1);
+  NO_BARRIER_FETCH_ADD(&g_memory_counters.total_size_absolute, (gpr_atm)size);
+  NO_BARRIER_FETCH_ADD(&g_memory_counters.total_size_relative, (gpr_atm)size);
+  NO_BARRIER_FETCH_ADD(&g_memory_counters.total_allocs_absolute, (gpr_atm)1);
+  NO_BARRIER_FETCH_ADD(&g_memory_counters.total_allocs_relative, (gpr_atm)1);
   ptr = g_old_allocs.malloc_fn(size + sizeof(size));
   *ptr++ = size;
   return ptr;
@@ -72,14 +78,10 @@ static void *guard_realloc(void *vptr, size_t size) {
     return NULL;
   }
   --ptr;
-  gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_absolute,
-                               (gpr_atm)size);
-  gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative,
-                               -(gpr_atm)*ptr);
-  gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative,
-                               (gpr_atm)size);
-  gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_absolute,
-                               (gpr_atm)1);
+  NO_BARRIER_FETCH_ADD(&g_memory_counters.total_size_absolute, (gpr_atm)size);
+  NO_BARRIER_FETCH_ADD(&g_memory_counters.total_size_relative, -(gpr_atm)*ptr);
+  NO_BARRIER_FETCH_ADD(&g_memory_counters.total_size_relative, (gpr_atm)size);
+  NO_BARRIER_FETCH_ADD(&g_memory_counters.total_allocs_absolute, (gpr_atm)1);
   ptr = g_old_allocs.realloc_fn(ptr, size + sizeof(size));
   *ptr++ = size;
   return ptr;
@@ -89,10 +91,8 @@ static void guard_free(void *vptr) {
   size_t *ptr = vptr;
   if (!vptr) return;
   --ptr;
-  gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_size_relative,
-                               -(gpr_atm)*ptr);
-  gpr_atm_no_barrier_fetch_add(&g_memory_counters.total_allocs_relative,
-                               -(gpr_atm)1);
+  NO_BARRIER_FETCH_ADD(&g_memory_counters.total_size_relative, -(gpr_atm)*ptr);
+  NO_BARRIER_FETCH_ADD(&g_memory_counters.total_allocs_relative, -(gpr_atm)1);
   g_old_allocs.free_fn(ptr);
 }
 
@@ -112,12 +112,12 @@ void grpc_memory_counters_destroy() {
 struct grpc_memory_counters grpc_memory_counters_snapshot() {
   struct grpc_memory_counters counters;
   counters.total_size_relative =
-      gpr_atm_no_barrier_load(&g_memory_counters.total_size_relative);
+      NO_BARRIER_LOAD(&g_memory_counters.total_size_relative);
   counters.total_size_absolute =
-      gpr_atm_no_barrier_load(&g_memory_counters.total_size_absolute);
+      NO_BARRIER_LOAD(&g_memory_counters.total_size_absolute);
   counters.total_allocs_relative =
-      gpr_atm_no_barrier_load(&g_memory_counters.total_allocs_relative);
+      NO_BARRIER_LOAD(&g_memory_counters.total_allocs_relative);
   counters.total_allocs_absolute =
-      gpr_atm_no_barrier_load(&g_memory_counters.total_allocs_absolute);
+      NO_BARRIER_LOAD(&g_memory_counters.total_allocs_absolute);
   return counters;
 }
diff --git a/test/core/util/trickle_endpoint.c b/test/core/util/trickle_endpoint.c
index 7ab0488a66..0848147158 100644
--- a/test/core/util/trickle_endpoint.c
+++ b/test/core/util/trickle_endpoint.c
@@ -31,6 +31,8 @@
  *
  */
 
+#include "src/core/lib/iomgr/sockaddr.h"
+
 #include "test/core/util/passthru_endpoint.h"
 
 #include <inttypes.h>
@@ -40,9 +42,6 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
-
-#include "src/core/lib/iomgr/sockaddr.h"
-
 #include "src/core/lib/slice/slice_internal.h"
 
 typedef struct {
diff --git a/test/cpp/common/channel_arguments_test.cc b/test/cpp/common/channel_arguments_test.cc
index 190d32ce06..9bcc9f99f6 100644
--- a/test/cpp/common/channel_arguments_test.cc
+++ b/test/cpp/common/channel_arguments_test.cc
@@ -230,13 +230,6 @@ TEST_F(ChannelArgumentsTest, SetSocketMutator) {
   EXPECT_TRUE(HasArg(arg1));
   // arg0 is replaced by arg1
   EXPECT_FALSE(HasArg(arg0));
-
-  // arg0 is destroyed by grpc_socket_mutator_to_arg(mutator1)
-  {
-    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-    arg1.value.pointer.vtable->destroy(&exec_ctx, arg1.value.pointer.p);
-    grpc_exec_ctx_finish(&exec_ctx);
-  }
 }
 
 TEST_F(ChannelArgumentsTest, SetUserAgentPrefix) {
diff --git a/test/cpp/interop/http2_client.cc b/test/cpp/interop/http2_client.cc
index 01c07823cf..38a437f39f 100644
--- a/test/cpp/interop/http2_client.cc
+++ b/test/cpp/interop/http2_client.cc
@@ -41,7 +41,7 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/transport/byte_stream.h"
-#include "src/proto/grpc/testing/messages.grpc.pb.h"
+#include "src/proto/grpc/testing/messages.pb.h"
 #include "src/proto/grpc/testing/test.grpc.pb.h"
 #include "test/cpp/interop/http2_client.h"
 
diff --git a/test/cpp/interop/http2_client.h b/test/cpp/interop/http2_client.h
index 12df5d26bc..e57d695205 100644
--- a/test/cpp/interop/http2_client.h
+++ b/test/cpp/interop/http2_client.h
@@ -38,7 +38,7 @@
 
 #include <grpc++/channel.h>
 #include <grpc/grpc.h>
-#include "src/proto/grpc/testing/messages.grpc.pb.h"
+#include "src/proto/grpc/testing/messages.pb.h"
 #include "src/proto/grpc/testing/test.grpc.pb.h"
 
 namespace grpc {
diff --git a/test/cpp/interop/interop_client.cc b/test/cpp/interop/interop_client.cc
index b7f2723c39..55ba324cc7 100644
--- a/test/cpp/interop/interop_client.cc
+++ b/test/cpp/interop/interop_client.cc
@@ -46,8 +46,8 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/transport/byte_stream.h"
-#include "src/proto/grpc/testing/empty.grpc.pb.h"
-#include "src/proto/grpc/testing/messages.grpc.pb.h"
+#include "src/proto/grpc/testing/empty.pb.h"
+#include "src/proto/grpc/testing/messages.pb.h"
 #include "src/proto/grpc/testing/test.grpc.pb.h"
 #include "test/cpp/interop/client_helper.h"
 #include "test/cpp/interop/interop_client.h"
diff --git a/test/cpp/interop/interop_client.h b/test/cpp/interop/interop_client.h
index 74f4db6b78..efcb7d2860 100644
--- a/test/cpp/interop/interop_client.h
+++ b/test/cpp/interop/interop_client.h
@@ -38,7 +38,7 @@
 
 #include <grpc++/channel.h>
 #include <grpc/grpc.h>
-#include "src/proto/grpc/testing/messages.grpc.pb.h"
+#include "src/proto/grpc/testing/messages.pb.h"
 #include "src/proto/grpc/testing/test.grpc.pb.h"
 
 namespace grpc {
diff --git a/test/cpp/interop/interop_server.cc b/test/cpp/interop/interop_server.cc
index 5a810b45ef..1cbca17928 100644
--- a/test/cpp/interop/interop_server.cc
+++ b/test/cpp/interop/interop_server.cc
@@ -48,8 +48,8 @@
 
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/byte_stream.h"
-#include "src/proto/grpc/testing/empty.grpc.pb.h"
-#include "src/proto/grpc/testing/messages.grpc.pb.h"
+#include "src/proto/grpc/testing/empty.pb.h"
+#include "src/proto/grpc/testing/messages.pb.h"
 #include "src/proto/grpc/testing/test.grpc.pb.h"
 #include "test/cpp/interop/server_helper.h"
 #include "test/cpp/util/test_config.h"
diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc
index 1c2f606637..01d985068d 100644
--- a/test/cpp/interop/reconnect_interop_client.cc
+++ b/test/cpp/interop/reconnect_interop_client.cc
@@ -40,8 +40,8 @@
 #include <grpc++/support/channel_arguments.h>
 #include <grpc/grpc.h>
 #include <grpc/support/log.h>
-#include "src/proto/grpc/testing/empty.grpc.pb.h"
-#include "src/proto/grpc/testing/messages.grpc.pb.h"
+#include "src/proto/grpc/testing/empty.pb.h"
+#include "src/proto/grpc/testing/messages.pb.h"
 #include "src/proto/grpc/testing/test.grpc.pb.h"
 #include "test/cpp/util/create_test_channel.h"
 #include "test/cpp/util/test_config.h"
diff --git a/test/cpp/interop/reconnect_interop_server.cc b/test/cpp/interop/reconnect_interop_server.cc
index 634d0a90fc..8d1b884af9 100644
--- a/test/cpp/interop/reconnect_interop_server.cc
+++ b/test/cpp/interop/reconnect_interop_server.cc
@@ -47,8 +47,8 @@
 #include <grpc/grpc.h>
 #include <grpc/support/log.h>
 
-#include "src/proto/grpc/testing/empty.grpc.pb.h"
-#include "src/proto/grpc/testing/messages.grpc.pb.h"
+#include "src/proto/grpc/testing/empty.pb.h"
+#include "src/proto/grpc/testing/messages.pb.h"
 #include "src/proto/grpc/testing/test.grpc.pb.h"
 #include "test/core/util/reconnect_server.h"
 #include "test/cpp/util/test_config.h"
diff --git a/test/cpp/microbenchmarks/bm_arena.cc b/test/cpp/microbenchmarks/bm_arena.cc
new file mode 100644
index 0000000000..770c0b6d47
--- /dev/null
+++ b/test/cpp/microbenchmarks/bm_arena.cc
@@ -0,0 +1,76 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* Benchmark arenas */
+
+extern "C" {
+#include "src/core/lib/support/arena.h"
+}
+#include "test/cpp/microbenchmarks/helpers.h"
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+
+static void BM_Arena_NoOp(benchmark::State& state) {
+  while (state.KeepRunning()) {
+    gpr_arena_destroy(gpr_arena_create(state.range(0)));
+  }
+}
+BENCHMARK(BM_Arena_NoOp)->Range(1, 1024 * 1024);
+
+static void BM_Arena_ManyAlloc(benchmark::State& state) {
+  gpr_arena* a = gpr_arena_create(state.range(0));
+  const size_t realloc_after =
+      1024 * 1024 * 1024 / ((state.range(1) + 15) & 0xffffff0u);
+  while (state.KeepRunning()) {
+    gpr_arena_alloc(a, state.range(1));
+    // periodically recreate arena to avoid OOM
+    if (state.iterations() % realloc_after == 0) {
+      gpr_arena_destroy(a);
+      a = gpr_arena_create(state.range(0));
+    }
+  }
+  gpr_arena_destroy(a);
+}
+BENCHMARK(BM_Arena_ManyAlloc)->Ranges({{1, 1024 * 1024}, {1, 32 * 1024}});
+
+static void BM_Arena_Batch(benchmark::State& state) {
+  while (state.KeepRunning()) {
+    gpr_arena* a = gpr_arena_create(state.range(0));
+    for (int i = 0; i < state.range(1); i++) {
+      gpr_arena_alloc(a, state.range(2));
+    }
+    gpr_arena_destroy(a);
+  }
+}
+BENCHMARK(BM_Arena_Batch)->Ranges({{1, 64 * 1024}, {1, 64}, {1, 1024}});
+
+BENCHMARK_MAIN();
diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc
index e3884785da..ec6073e786 100644
--- a/test/cpp/microbenchmarks/bm_call_create.cc
+++ b/test/cpp/microbenchmarks/bm_call_create.cc
@@ -37,6 +37,7 @@
 #include <string.h>
 #include <sstream>
 
+#include <grpc++/channel.h>
 #include <grpc++/support/channel_arguments.h>
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
@@ -55,11 +56,35 @@ extern "C" {
 #include "src/core/lib/transport/transport_impl.h"
 }
 
+#include "src/cpp/client/create_channel_internal.h"
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
 #include "test/cpp/microbenchmarks/helpers.h"
 #include "third_party/benchmark/include/benchmark/benchmark.h"
 
 auto &force_library_initialization = Library::get();
 
+void BM_Zalloc(benchmark::State &state) {
+  // speed of light for call creation is zalloc, so benchmark a few interesting
+  // sizes
+  size_t sz = state.range(0);
+  while (state.KeepRunning()) {
+    gpr_free(gpr_zalloc(sz));
+  }
+}
+BENCHMARK(BM_Zalloc)
+    ->Arg(64)
+    ->Arg(128)
+    ->Arg(256)
+    ->Arg(512)
+    ->Arg(1024)
+    ->Arg(1536)
+    ->Arg(2048)
+    ->Arg(3072)
+    ->Arg(4096)
+    ->Arg(5120)
+    ->Arg(6144)
+    ->Arg(7168);
+
 class BaseChannelFixture {
  public:
   BaseChannelFixture(grpc_channel *channel) : channel_(channel) {}
@@ -106,6 +131,33 @@ static void BM_CallCreateDestroy(benchmark::State &state) {
 BENCHMARK_TEMPLATE(BM_CallCreateDestroy, InsecureChannel);
 BENCHMARK_TEMPLATE(BM_CallCreateDestroy, LameChannel);
 
+static void *tag(int i) {
+  return reinterpret_cast<void *>(static_cast<intptr_t>(i));
+}
+
+static void BM_LameChannelCallCreateCpp(benchmark::State &state) {
+  TrackCounters track_counters;
+  auto stub =
+      grpc::testing::EchoTestService::NewStub(grpc::CreateChannelInternal(
+          "", grpc_lame_client_channel_create(
+                  "localhost:1234", GRPC_STATUS_UNAUTHENTICATED, "blah")));
+  grpc::CompletionQueue cq;
+  grpc::testing::EchoRequest send_request;
+  grpc::testing::EchoResponse recv_response;
+  grpc::Status recv_status;
+  while (state.KeepRunning()) {
+    grpc::ClientContext cli_ctx;
+    auto reader = stub->AsyncEcho(&cli_ctx, send_request, &cq);
+    reader->Finish(&recv_response, &recv_status, tag(0));
+    void *t;
+    bool ok;
+    GPR_ASSERT(cq.Next(&t, &ok));
+    GPR_ASSERT(ok);
+  }
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_LameChannelCallCreateCpp);
+
 static void FilterDestroy(grpc_exec_ctx *exec_ctx, void *arg,
                           grpc_error *error) {
   gpr_free(arg);
@@ -181,7 +233,7 @@ static void SetPollsetOrPollsetSet(grpc_exec_ctx *exec_ctx,
 
 static void DestroyCallElem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
                             const grpc_call_final_info *final_info,
-                            void *and_free_memory) {}
+                            grpc_closure *then_sched_closure) {}
 
 grpc_error *InitChannelElem(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem,
                             grpc_channel_element_args *args) {
@@ -224,7 +276,7 @@ const char *name;
 /* implementation of grpc_transport_init_stream */
 int InitStream(grpc_exec_ctx *exec_ctx, grpc_transport *self,
                grpc_stream *stream, grpc_stream_refcount *refcount,
-               const void *server_data) {
+               const void *server_data, gpr_arena *arena) {
   return 0;
 }
 
@@ -248,7 +300,7 @@ void PerformOp(grpc_exec_ctx *exec_ctx, grpc_transport *self,
 
 /* implementation of grpc_transport_destroy_stream */
 void DestroyStream(grpc_exec_ctx *exec_ctx, grpc_transport *self,
-                   grpc_stream *stream, void *and_free_memory) {}
+                   grpc_stream *stream, grpc_closure *then_sched_closure) {}
 
 /* implementation of grpc_transport_destroy */
 void Destroy(grpc_exec_ctx *exec_ctx, grpc_transport *self) {}
@@ -343,7 +395,7 @@ static void BM_IsolatedFilter(benchmark::State &state) {
   grpc_channel_stack *channel_stack =
       static_cast<grpc_channel_stack *>(gpr_zalloc(channel_size));
   GPR_ASSERT(GRPC_LOG_IF_ERROR(
-      "call_stack_init",
+      "channel_stack_init",
       grpc_channel_stack_init(&exec_ctx, 1, FilterDestroy, channel_stack,
                               &filters[0], filters.size(), &channel_args,
                               fixture.flags & REQUIRES_TRANSPORT
@@ -358,15 +410,29 @@ static void BM_IsolatedFilter(benchmark::State &state) {
   grpc_slice method = grpc_slice_from_static_string("/foo/bar");
   grpc_call_final_info final_info;
   TestOp test_op_data;
+  grpc_call_element_args call_args;
+  call_args.call_stack = call_stack;
+  call_args.server_transport_data = NULL;
+  call_args.context = NULL;
+  call_args.path = method;
+  call_args.start_time = start_time;
+  call_args.deadline = deadline;
+  const int kArenaSize = 4096;
+  call_args.arena = gpr_arena_create(kArenaSize);
   while (state.KeepRunning()) {
     GRPC_ERROR_UNREF(grpc_call_stack_init(&exec_ctx, channel_stack, 1,
-                                          DoNothing, NULL, NULL, NULL, method,
-                                          start_time, deadline, call_stack));
+                                          DoNothing, NULL, &call_args));
     typename TestOp::Op op(&exec_ctx, &test_op_data, call_stack);
     grpc_call_stack_destroy(&exec_ctx, call_stack, &final_info, NULL);
     op.Finish(&exec_ctx);
     grpc_exec_ctx_flush(&exec_ctx);
+    // recreate arena every 64k iterations to avoid oom
+    if (0 == (state.iterations() & 0xffff)) {
+      gpr_arena_destroy(call_args.arena);
+      call_args.arena = gpr_arena_create(kArenaSize);
+    }
   }
+  gpr_arena_destroy(call_args.arena);
   grpc_channel_stack_destroy(&exec_ctx, channel_stack);
   grpc_exec_ctx_finish(&exec_ctx);
   gpr_free(channel_stack);
diff --git a/test/cpp/microbenchmarks/bm_error.cc b/test/cpp/microbenchmarks/bm_error.cc
index 95f19e7586..c4f6aa19d5 100644
--- a/test/cpp/microbenchmarks/bm_error.cc
+++ b/test/cpp/microbenchmarks/bm_error.cc
@@ -71,6 +71,42 @@ static void BM_ErrorCreateAndSetStatus(benchmark::State& state) {
 }
 BENCHMARK(BM_ErrorCreateAndSetStatus);
 
+static void BM_ErrorCreateAndSetIntAndStr(benchmark::State& state) {
+  TrackCounters track_counters;
+  while (state.KeepRunning()) {
+    GRPC_ERROR_UNREF(grpc_error_set_str(
+        grpc_error_set_int(GRPC_ERROR_CREATE("GOAWAY received"),
+                           GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)0),
+        GRPC_ERROR_STR_RAW_BYTES, "raw bytes"));
+  }
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_ErrorCreateAndSetIntAndStr);
+
+static void BM_ErrorCreateAndSetIntLoop(benchmark::State& state) {
+  TrackCounters track_counters;
+  grpc_error* error = GRPC_ERROR_CREATE("Error");
+  int n = 0;
+  while (state.KeepRunning()) {
+    error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, n++);
+  }
+  GRPC_ERROR_UNREF(error);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_ErrorCreateAndSetIntLoop);
+
+static void BM_ErrorCreateAndSetStrLoop(benchmark::State& state) {
+  TrackCounters track_counters;
+  grpc_error* error = GRPC_ERROR_CREATE("Error");
+  const char* str = "hello";
+  while (state.KeepRunning()) {
+    error = grpc_error_set_str(error, GRPC_ERROR_STR_GRPC_MESSAGE, str);
+  }
+  GRPC_ERROR_UNREF(error);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_ErrorCreateAndSetStrLoop);
+
 static void BM_ErrorRefUnref(benchmark::State& state) {
   TrackCounters track_counters;
   grpc_error* error = GRPC_ERROR_CREATE("Error");
diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc
index dc0e7d769a..00e37f7912 100644
--- a/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc
+++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc
@@ -54,86 +54,141 @@ auto& force_library_initialization = Library::get();
 
 static void* tag(intptr_t x) { return reinterpret_cast<void*>(x); }
 
-template <class Fixture>
-static void BM_PumpStreamClientToServer(benchmark::State& state) {
+// Repeatedly makes Streaming Bidi calls (exchanging a configurable number of
+// messages in each call) in a loop on a single channel
+//
+//  First parmeter (i.e state.range(0)):  Message size (in bytes) to use
+//  Second parameter (i.e state.range(1)): Number of ping pong messages.
+//      Note: One ping-pong means two messages (one from client to server and
+//      the other from server to client):
+template <class Fixture, class ClientContextMutator, class ServerContextMutator>
+static void BM_StreamingPingPong(benchmark::State& state) {
+  const int msg_size = state.range(0);
+  const int max_ping_pongs = state.range(1);
+
   EchoTestService::AsyncService service;
   std::unique_ptr<Fixture> fixture(new Fixture(&service));
   {
+    EchoResponse send_response;
+    EchoResponse recv_response;
     EchoRequest send_request;
     EchoRequest recv_request;
-    if (state.range(0) > 0) {
-      send_request.set_message(std::string(state.range(0), 'a'));
+
+    if (msg_size > 0) {
+      send_request.set_message(std::string(msg_size, 'a'));
+      send_response.set_message(std::string(msg_size, 'b'));
     }
-    Status recv_status;
-    ServerContext svr_ctx;
-    ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
-    service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
-                              fixture->cq(), tag(0));
+
     std::unique_ptr<EchoTestService::Stub> stub(
         EchoTestService::NewStub(fixture->channel()));
-    ClientContext cli_ctx;
-    auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
-    int need_tags = (1 << 0) | (1 << 1);
-    void* t;
-    bool ok;
-    while (need_tags) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      GPR_ASSERT(ok);
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
-    }
-    response_rw.Read(&recv_request, tag(0));
+
     while (state.KeepRunning()) {
-      GPR_TIMER_SCOPE("BenchmarkCycle", 0);
-      request_rw->Write(send_request, tag(1));
-      while (true) {
+      ServerContext svr_ctx;
+      ServerContextMutator svr_ctx_mut(&svr_ctx);
+      ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
+      service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
+                                fixture->cq(), tag(0));
+
+      ClientContext cli_ctx;
+      ClientContextMutator cli_ctx_mut(&cli_ctx);
+      auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
+
+      // Establish async stream between client side and server side
+      void* t;
+      bool ok;
+      int need_tags = (1 << 0) | (1 << 1);
+      while (need_tags) {
         GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-        if (t == tag(0)) {
-          response_rw.Read(&recv_request, tag(0));
-        } else if (t == tag(1)) {
-          break;
-        } else {
-          GPR_ASSERT(false);
+        GPR_ASSERT(ok);
+        int i = (int)(intptr_t)t;
+        GPR_ASSERT(need_tags & (1 << i));
+        need_tags &= ~(1 << i);
+      }
+
+      // Send 'max_ping_pongs' number of ping pong messages
+      int ping_pong_cnt = 0;
+      while (ping_pong_cnt < max_ping_pongs) {
+        request_rw->Write(send_request, tag(0));   // Start client send
+        response_rw.Read(&recv_request, tag(1));   // Start server recv
+        request_rw->Read(&recv_response, tag(2));  // Start client recv
+
+        need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3);
+        while (need_tags) {
+          GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+          GPR_ASSERT(ok);
+          int i = (int)(intptr_t)t;
+
+          // If server recv is complete, start the server send operation
+          if (i == 1) {
+            response_rw.Write(send_response, tag(3));
+          }
+
+          GPR_ASSERT(need_tags & (1 << i));
+          need_tags &= ~(1 << i);
         }
+
+        ping_pong_cnt++;
       }
-    }
-    request_rw->WritesDone(tag(1));
-    need_tags = (1 << 0) | (1 << 1);
-    while (need_tags) {
-      GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-      int i = (int)(intptr_t)t;
-      GPR_ASSERT(need_tags & (1 << i));
-      need_tags &= ~(1 << i);
+
+      request_rw->WritesDone(tag(0));
+      response_rw.Finish(Status::OK, tag(1));
+
+      Status recv_status;
+      request_rw->Finish(&recv_status, tag(2));
+
+      need_tags = (1 << 0) | (1 << 1) | (1 << 2);
+      while (need_tags) {
+        GPR_ASSERT(fixture->cq()->Next(&t, &ok));
+        int i = (int)(intptr_t)t;
+        GPR_ASSERT(need_tags & (1 << i));
+        need_tags &= ~(1 << i);
+      }
+
+      GPR_ASSERT(recv_status.ok());
     }
   }
+
   fixture->Finish(state);
   fixture.reset();
-  state.SetBytesProcessed(state.range(0) * state.iterations());
+  state.SetBytesProcessed(msg_size * state.iterations() * max_ping_pongs * 2);
 }
 
-template <class Fixture>
-static void BM_PumpStreamServerToClient(benchmark::State& state) {
+// Repeatedly sends ping pong messages in a single streaming Bidi call in a loop
+//     First parmeter (i.e state.range(0)):  Message size (in bytes) to use
+template <class Fixture, class ClientContextMutator, class ServerContextMutator>
+static void BM_StreamingPingPongMsgs(benchmark::State& state) {
+  const int msg_size = state.range(0);
+
   EchoTestService::AsyncService service;
   std::unique_ptr<Fixture> fixture(new Fixture(&service));
   {
     EchoResponse send_response;
     EchoResponse recv_response;
-    if (state.range(0) > 0) {
-      send_response.set_message(std::string(state.range(0), 'a'));
+    EchoRequest send_request;
+    EchoRequest recv_request;
+
+    if (msg_size > 0) {
+      send_request.set_message(std::string(msg_size, 'a'));
+      send_response.set_message(std::string(msg_size, 'b'));
     }
-    Status recv_status;
+
+    std::unique_ptr<EchoTestService::Stub> stub(
+        EchoTestService::NewStub(fixture->channel()));
+
     ServerContext svr_ctx;
+    ServerContextMutator svr_ctx_mut(&svr_ctx);
     ServerAsyncReaderWriter<EchoResponse, EchoRequest> response_rw(&svr_ctx);
     service.RequestBidiStream(&svr_ctx, &response_rw, fixture->cq(),
                               fixture->cq(), tag(0));
-    std::unique_ptr<EchoTestService::Stub> stub(
-        EchoTestService::NewStub(fixture->channel()));
+
     ClientContext cli_ctx;
+    ClientContextMutator cli_ctx_mut(&cli_ctx);
     auto request_rw = stub->AsyncBidiStream(&cli_ctx, fixture->cq(), tag(1));
-    int need_tags = (1 << 0) | (1 << 1);
+
+    // Establish async stream between client side and server side
     void* t;
     bool ok;
+    int need_tags = (1 << 0) | (1 << 1);
     while (need_tags) {
       GPR_ASSERT(fixture->cq()->Next(&t, &ok));
       GPR_ASSERT(ok);
@@ -141,54 +196,78 @@ static void BM_PumpStreamServerToClient(benchmark::State& state) {
       GPR_ASSERT(need_tags & (1 << i));
       need_tags &= ~(1 << i);
     }
-    request_rw->Read(&recv_response, tag(0));
+
     while (state.KeepRunning()) {
       GPR_TIMER_SCOPE("BenchmarkCycle", 0);
-      response_rw.Write(send_response, tag(1));
-      while (true) {
+      request_rw->Write(send_request, tag(0));   // Start client send
+      response_rw.Read(&recv_request, tag(1));   // Start server recv
+      request_rw->Read(&recv_response, tag(2));  // Start client recv
+
+      need_tags = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3);
+      while (need_tags) {
         GPR_ASSERT(fixture->cq()->Next(&t, &ok));
-        if (t == tag(0)) {
-          request_rw->Read(&recv_response, tag(0));
-        } else if (t == tag(1)) {
-          break;
-        } else {
-          GPR_ASSERT(false);
+        GPR_ASSERT(ok);
+        int i = (int)(intptr_t)t;
+
+        // If server recv is complete, start the server send operation
+        if (i == 1) {
+          response_rw.Write(send_response, tag(3));
         }
+
+        GPR_ASSERT(need_tags & (1 << i));
+        need_tags &= ~(1 << i);
       }
     }
+
+    request_rw->WritesDone(tag(0));
     response_rw.Finish(Status::OK, tag(1));
-    need_tags = (1 << 0) | (1 << 1);
+    Status recv_status;
+    request_rw->Finish(&recv_status, tag(2));
+
+    need_tags = (1 << 0) | (1 << 1) | (1 << 2);
     while (need_tags) {
       GPR_ASSERT(fixture->cq()->Next(&t, &ok));
       int i = (int)(intptr_t)t;
       GPR_ASSERT(need_tags & (1 << i));
       need_tags &= ~(1 << i);
     }
+
+    GPR_ASSERT(recv_status.ok());
   }
+
   fixture->Finish(state);
   fixture.reset();
-  state.SetBytesProcessed(state.range(0) * state.iterations());
+  state.SetBytesProcessed(msg_size * state.iterations() * 2);
 }
 
 /*******************************************************************************
  * CONFIGURATIONS
  */
 
-BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS)
-    ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair)
+// Generate Args for StreamingPingPong benchmarks. Currently generates args for
+// only "small streams" (i.e streams with 0, 1 or 2 messages)
+static void StreamingPingPongArgs(benchmark::internal::Benchmark* b) {
+  int msg_size = 0;
+
+  b->Args({0, 0});  // spl case: 0 ping-pong msgs (msg_size doesn't matter here)
+
+  for (msg_size = 0; msg_size <= 128 * 1024 * 1024;
+       msg_size == 0 ? msg_size++ : msg_size *= 8) {
+    b->Args({msg_size, 1});
+    b->Args({msg_size, 2});
+  }
+}
+
+BENCHMARK_TEMPLATE(BM_StreamingPingPong, InProcessCHTTP2, NoOpMutator,
+                   NoOpMutator)
+    ->Apply(StreamingPingPongArgs);
+BENCHMARK_TEMPLATE(BM_StreamingPingPong, TCP, NoOpMutator, NoOpMutator)
+    ->Apply(StreamingPingPongArgs);
+
+BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, InProcessCHTTP2, NoOpMutator,
+                   NoOpMutator)
     ->Range(0, 128 * 1024 * 1024);
-BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2)
+BENCHMARK_TEMPLATE(BM_StreamingPingPongMsgs, TCP, NoOpMutator, NoOpMutator)
     ->Range(0, 128 * 1024 * 1024);
 
 }  // namespace testing
diff --git a/test/cpp/microbenchmarks/bm_metadata.cc b/test/cpp/microbenchmarks/bm_metadata.cc
index ee3dec2bce..34874b57f5 100644
--- a/test/cpp/microbenchmarks/bm_metadata.cc
+++ b/test/cpp/microbenchmarks/bm_metadata.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2017, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc
new file mode 100644
index 0000000000..0f3d3cef66
--- /dev/null
+++ b/test/cpp/microbenchmarks/bm_pollset.cc
@@ -0,0 +1,254 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/* Test out pollset latencies */
+
+#include <grpc/grpc.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/useful.h>
+
+extern "C" {
+#include "src/core/lib/iomgr/ev_posix.h"
+#include "src/core/lib/iomgr/pollset.h"
+#include "src/core/lib/iomgr/port.h"
+#include "src/core/lib/iomgr/wakeup_fd_posix.h"
+}
+
+#include "test/cpp/microbenchmarks/helpers.h"
+#include "third_party/benchmark/include/benchmark/benchmark.h"
+
+#include <string.h>
+
+#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <unistd.h>
+#endif
+
+auto& force_library_initialization = Library::get();
+
+static void shutdown_ps(grpc_exec_ctx* exec_ctx, void* ps, grpc_error* error) {
+  grpc_pollset_destroy(static_cast<grpc_pollset*>(ps));
+}
+
+static void BM_CreateDestroyPollset(benchmark::State& state) {
+  TrackCounters track_counters;
+  size_t ps_sz = grpc_pollset_size();
+  grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_malloc(ps_sz));
+  gpr_mu* mu;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_closure shutdown_ps_closure;
+  grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps,
+                    grpc_schedule_on_exec_ctx);
+  while (state.KeepRunning()) {
+    memset(ps, 0, ps_sz);
+    grpc_pollset_init(ps, &mu);
+    gpr_mu_lock(mu);
+    grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure);
+    gpr_mu_unlock(mu);
+    grpc_exec_ctx_flush(&exec_ctx);
+  }
+  grpc_exec_ctx_finish(&exec_ctx);
+  gpr_free(ps);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_CreateDestroyPollset);
+
+#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL
+static void BM_PollEmptyPollset_SpeedOfLight(benchmark::State& state) {
+  // equivalent to BM_PollEmptyPollset, but just use the OS primitives to guage
+  // what the speed of light would be if we abstracted perfectly
+  TrackCounters track_counters;
+  int epfd = epoll_create1(0);
+  GPR_ASSERT(epfd != -1);
+  size_t nev = state.range(0);
+  size_t nfd = state.range(1);
+  epoll_event* ev = new epoll_event[nev];
+  std::vector<int> fds;
+  for (size_t i = 0; i < nfd; i++) {
+    fds.push_back(eventfd(0, 0));
+    epoll_event ev;
+    ev.events = EPOLLIN;
+    epoll_ctl(epfd, EPOLL_CTL_ADD, fds.back(), &ev);
+  }
+  while (state.KeepRunning()) {
+    epoll_wait(epfd, ev, nev, 0);
+  }
+  for (auto fd : fds) {
+    close(fd);
+  }
+  close(epfd);
+  delete[] ev;
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_PollEmptyPollset_SpeedOfLight)
+    ->Args({1, 0})
+    ->Args({1, 1})
+    ->Args({1, 10})
+    ->Args({1, 100})
+    ->Args({1, 1000})
+    ->Args({1, 10000})
+    ->Args({1, 100000})
+    ->Args({10, 1})
+    ->Args({100, 1})
+    ->Args({1000, 1});
+#endif
+
+static void BM_PollEmptyPollset(benchmark::State& state) {
+  TrackCounters track_counters;
+  size_t ps_sz = grpc_pollset_size();
+  grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_zalloc(ps_sz));
+  gpr_mu* mu;
+  grpc_pollset_init(ps, &mu);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  gpr_timespec now = gpr_time_0(GPR_CLOCK_MONOTONIC);
+  gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC);
+  gpr_mu_lock(mu);
+  while (state.KeepRunning()) {
+    grpc_pollset_worker* worker;
+    GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, &worker, now, deadline));
+  }
+  grpc_closure shutdown_ps_closure;
+  grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps,
+                    grpc_schedule_on_exec_ctx);
+  grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure);
+  gpr_mu_unlock(mu);
+  grpc_exec_ctx_finish(&exec_ctx);
+  gpr_free(ps);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_PollEmptyPollset);
+
+class Closure : public grpc_closure {
+ public:
+  virtual ~Closure() {}
+};
+
+template <class F>
+Closure* MakeClosure(F f, grpc_closure_scheduler* scheduler) {
+  struct C : public Closure {
+    C(F f, grpc_closure_scheduler* scheduler) : f_(f) {
+      grpc_closure_init(this, C::cbfn, this, scheduler);
+    }
+    static void cbfn(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {
+      C* p = static_cast<C*>(arg);
+      p->f_();
+    }
+    F f_;
+  };
+  return new C(f, scheduler);
+}
+
+#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL
+static void BM_SingleThreadPollOneFd_SpeedOfLight(benchmark::State& state) {
+  // equivalent to BM_PollEmptyPollset, but just use the OS primitives to guage
+  // what the speed of light would be if we abstracted perfectly
+  TrackCounters track_counters;
+  int epfd = epoll_create1(0);
+  GPR_ASSERT(epfd != -1);
+  epoll_event ev[100];
+  int fd = eventfd(0, EFD_NONBLOCK);
+  ev[0].events = EPOLLIN;
+  epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev[0]);
+  while (state.KeepRunning()) {
+    int err;
+    do {
+      err = eventfd_write(fd, 1);
+    } while (err < 0 && errno == EINTR);
+    GPR_ASSERT(err == 0);
+    do {
+      err = epoll_wait(epfd, ev, GPR_ARRAY_SIZE(ev), 0);
+    } while (err < 0 && errno == EINTR);
+    GPR_ASSERT(err == 1);
+    eventfd_t value;
+    do {
+      err = eventfd_read(fd, &value);
+    } while (err < 0 && errno == EINTR);
+    GPR_ASSERT(err == 0);
+  }
+  close(fd);
+  close(epfd);
+  track_counters.Finish(state);
+}
+BENCHMARK(BM_SingleThreadPollOneFd_SpeedOfLight);
+#endif
+
+static void BM_SingleThreadPollOneFd(benchmark::State& state) {
+  TrackCounters track_counters;
+  size_t ps_sz = grpc_pollset_size();
+  grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_zalloc(ps_sz));
+  gpr_mu* mu;
+  grpc_pollset_init(ps, &mu);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  gpr_timespec now = gpr_time_0(GPR_CLOCK_MONOTONIC);
+  gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
+  grpc_wakeup_fd wakeup_fd;
+  GRPC_ERROR_UNREF(grpc_wakeup_fd_init(&wakeup_fd));
+  grpc_fd* wakeup = grpc_fd_create(wakeup_fd.read_fd, "wakeup_read");
+  grpc_pollset_add_fd(&exec_ctx, ps, wakeup);
+  bool done = false;
+  Closure* continue_closure = MakeClosure(
+      [&]() {
+        GRPC_ERROR_UNREF(grpc_wakeup_fd_consume_wakeup(&wakeup_fd));
+        if (!state.KeepRunning()) {
+          done = true;
+          return;
+        }
+        GRPC_ERROR_UNREF(grpc_wakeup_fd_wakeup(&wakeup_fd));
+        grpc_fd_notify_on_read(&exec_ctx, wakeup, continue_closure);
+      },
+      grpc_schedule_on_exec_ctx);
+  GRPC_ERROR_UNREF(grpc_wakeup_fd_wakeup(&wakeup_fd));
+  grpc_fd_notify_on_read(&exec_ctx, wakeup, continue_closure);
+  gpr_mu_lock(mu);
+  while (!done) {
+    grpc_pollset_worker* worker;
+    GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, &worker, now, deadline));
+  }
+  grpc_fd_orphan(&exec_ctx, wakeup, NULL, NULL, "done");
+  wakeup_fd.read_fd = 0;
+  grpc_closure shutdown_ps_closure;
+  grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps,
+                    grpc_schedule_on_exec_ctx);
+  grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure);
+  gpr_mu_unlock(mu);
+  grpc_exec_ctx_finish(&exec_ctx);
+  grpc_wakeup_fd_destroy(&wakeup_fd);
+  gpr_free(ps);
+  track_counters.Finish(state);
+  delete continue_closure;
+}
+BENCHMARK(BM_SingleThreadPollOneFd);
+
+BENCHMARK_MAIN();
diff --git a/test/cpp/microbenchmarks/helpers.cc b/test/cpp/microbenchmarks/helpers.cc
index 947e81ffd8..d277c5984c 100644
--- a/test/cpp/microbenchmarks/helpers.cc
+++ b/test/cpp/microbenchmarks/helpers.cc
@@ -45,6 +45,7 @@ void TrackCounters::Finish(benchmark::State &state) {
 
 void TrackCounters::AddToLabel(std::ostream &out, benchmark::State &state) {
 #ifdef GPR_LOW_LEVEL_COUNTERS
+  grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot();
   out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) -
                                      mu_locks_at_start_) /
                             (double)state.iterations())
@@ -55,11 +56,10 @@ void TrackCounters::AddToLabel(std::ostream &out, benchmark::State &state) {
       << " atm_add/iter:"
       << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) -
                    atm_add_at_start_) /
-          (double)state.iterations());
-#endif
-  grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot();
-  out << " allocs/iter:"
+          (double)state.iterations())
+      << " allocs/iter:"
       << ((double)(counters_at_end.total_allocs_absolute -
                    counters_at_start_.total_allocs_absolute) /
           (double)state.iterations());
+#endif
 }
diff --git a/test/cpp/microbenchmarks/helpers.h b/test/cpp/microbenchmarks/helpers.h
index 42a8fbaf0b..f44b7cf83a 100644
--- a/test/cpp/microbenchmarks/helpers.h
+++ b/test/cpp/microbenchmarks/helpers.h
@@ -55,7 +55,9 @@ class Library {
 
  private:
   Library() {
+#ifdef GPR_LOW_LEVEL_COUNTERS
     grpc_memory_counters_init();
+#endif
     init_lib_.init();
     rq_ = grpc_resource_quota_create("bm");
   }
@@ -84,8 +86,8 @@ class TrackCounters {
       gpr_atm_no_barrier_load(&gpr_counter_atm_cas);
   const size_t atm_add_at_start_ =
       gpr_atm_no_barrier_load(&gpr_counter_atm_add);
-#endif
   grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot();
+#endif
 };
 
 #endif
diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h
index baa9304cc2..25a19a5a74 100644
--- a/test/cpp/qps/client.h
+++ b/test/cpp/qps/client.h
@@ -46,7 +46,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 
-#include "src/proto/grpc/testing/payloads.grpc.pb.h"
+#include "src/proto/grpc/testing/payloads.pb.h"
 #include "src/proto/grpc/testing/services.grpc.pb.h"
 
 #include "test/cpp/qps/histogram.h"
diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc
index 4032039ea1..396d308e2a 100644
--- a/test/cpp/qps/client_async.cc
+++ b/test/cpp/qps/client_async.cc
@@ -112,7 +112,9 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext {
         next_state_ = State::RESP_DONE;
         return true;
       case State::RESP_DONE:
-        entry->set_value((UsageTimer::Now() - start_) * 1e9);
+        if (status_.ok()) {
+          entry->set_value((UsageTimer::Now() - start_) * 1e9);
+        }
         callback_(status_, &response_, entry);
         next_state_ = State::INVALID;
         return false;
diff --git a/test/cpp/qps/client_sync.cc b/test/cpp/qps/client_sync.cc
index 498416c64a..a944c45496 100644
--- a/test/cpp/qps/client_sync.cc
+++ b/test/cpp/qps/client_sync.cc
@@ -129,7 +129,9 @@ class SynchronousUnaryClient final : public SynchronousClient {
     grpc::ClientContext context;
     grpc::Status s =
         stub->UnaryCall(&context, request_, &responses_[thread_idx]);
-    entry->set_value((UsageTimer::Now() - start) * 1e9);
+    if (s.ok()) {
+      entry->set_value((UsageTimer::Now() - start) * 1e9);
+    }
     entry->set_status(s.error_code());
     return true;
   }
@@ -154,7 +156,7 @@ class SynchronousStreamingClient final : public SynchronousClient {
         (*stream)->WritesDone();
         Status s = (*stream)->Finish();
         if (!s.ok()) {
-          gpr_log(GPR_ERROR, "Stream %zu received an error %s", i,
+          gpr_log(GPR_ERROR, "Stream %" PRIuPTR " received an error %s", i,
                   s.error_message().c_str());
         }
       }
@@ -170,8 +172,17 @@ class SynchronousStreamingClient final : public SynchronousClient {
     if (stream_[thread_idx]->Write(request_) &&
         stream_[thread_idx]->Read(&responses_[thread_idx])) {
       entry->set_value((UsageTimer::Now() - start) * 1e9);
+      // don't set the status since there isn't one yet
       return true;
     }
+    stream_[thread_idx]->WritesDone();
+    Status s = stream_[thread_idx]->Finish();
+    // don't set the value since the stream is failed and shouldn't be timed
+    entry->set_status(s.error_code());
+    if (!s.ok()) {
+      gpr_log(GPR_ERROR, "Stream %" PRIuPTR " received an error %s", thread_idx,
+              s.error_message().c_str());
+    }
     auto* stub = channels_[thread_idx % channels_.size()].get_stub();
     context_[thread_idx].~ClientContext();
     new (&context_[thread_idx]) ClientContext();
diff --git a/test/cpp/qps/driver.h b/test/cpp/qps/driver.h
index e72d30a4ef..dd32a16c87 100644
--- a/test/cpp/qps/driver.h
+++ b/test/cpp/qps/driver.h
@@ -36,7 +36,7 @@
 
 #include <memory>
 
-#include "src/proto/grpc/testing/control.grpc.pb.h"
+#include "src/proto/grpc/testing/control.pb.h"
 #include "test/cpp/qps/histogram.h"
 
 namespace grpc {
diff --git a/test/cpp/qps/histogram.h b/test/cpp/qps/histogram.h
index acb415f0a1..470a394301 100644
--- a/test/cpp/qps/histogram.h
+++ b/test/cpp/qps/histogram.h
@@ -35,7 +35,7 @@
 #define TEST_QPS_HISTOGRAM_H
 
 #include <grpc/support/histogram.h>
-#include "src/proto/grpc/testing/stats.grpc.pb.h"
+#include "src/proto/grpc/testing/stats.pb.h"
 
 namespace grpc {
 namespace testing {
diff --git a/test/cpp/qps/qps_json_driver.cc b/test/cpp/qps/qps_json_driver.cc
index ddaaa7ca75..bd2c1f0ec6 100644
--- a/test/cpp/qps/qps_json_driver.cc
+++ b/test/cpp/qps/qps_json_driver.cc
@@ -31,6 +31,7 @@
  *
  */
 
+#include <iostream>
 #include <memory>
 #include <set>
 
diff --git a/test/cpp/qps/server.h b/test/cpp/qps/server.h
index 821d5935be..8fbf37a095 100644
--- a/test/cpp/qps/server.h
+++ b/test/cpp/qps/server.h
@@ -38,8 +38,8 @@
 #include <grpc/support/cpu.h>
 #include <vector>
 
-#include "src/proto/grpc/testing/control.grpc.pb.h"
-#include "src/proto/grpc/testing/messages.grpc.pb.h"
+#include "src/proto/grpc/testing/control.pb.h"
+#include "src/proto/grpc/testing/messages.pb.h"
 #include "test/core/end2end/data/ssl_test_data.h"
 #include "test/core/util/port.h"
 #include "test/cpp/qps/usage_timer.h"
diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc
index b3a06aeaf5..b58d91eea6 100644
--- a/test/cpp/qps/server_async.cc
+++ b/test/cpp/qps/server_async.cc
@@ -103,24 +103,25 @@ class AsyncQpsServerTest final : public grpc::testing::Server {
 
     server_ = builder.BuildAndStart();
 
-    using namespace std::placeholders;
-
     auto process_rpc_bound =
-        std::bind(process_rpc, config.payload_config(), _1, _2);
+        std::bind(process_rpc, config.payload_config(), std::placeholders::_1,
+                  std::placeholders::_2);
 
     for (int i = 0; i < 15000; i++) {
       for (int j = 0; j < num_threads; j++) {
         if (request_unary_function) {
-          auto request_unary =
-              std::bind(request_unary_function, &async_service_, _1, _2, _3,
-                        srv_cqs_[j].get(), srv_cqs_[j].get(), _4);
+          auto request_unary = std::bind(
+              request_unary_function, &async_service_, std::placeholders::_1,
+              std::placeholders::_2, std::placeholders::_3, srv_cqs_[j].get(),
+              srv_cqs_[j].get(), std::placeholders::_4);
           contexts_.emplace_back(
               new ServerRpcContextUnaryImpl(request_unary, process_rpc_bound));
         }
         if (request_streaming_function) {
-          auto request_streaming =
-              std::bind(request_streaming_function, &async_service_, _1, _2,
-                        srv_cqs_[j].get(), srv_cqs_[j].get(), _3);
+          auto request_streaming = std::bind(
+              request_streaming_function, &async_service_,
+              std::placeholders::_1, std::placeholders::_2, srv_cqs_[j].get(),
+              srv_cqs_[j].get(), std::placeholders::_3);
           contexts_.emplace_back(new ServerRpcContextStreamingImpl(
               request_streaming, process_rpc_bound));
         }
diff --git a/test/cpp/qps/server_sync.cc b/test/cpp/qps/server_sync.cc
index 8076a4a6b9..f79284d225 100644
--- a/test/cpp/qps/server_sync.cc
+++ b/test/cpp/qps/server_sync.cc
@@ -74,7 +74,9 @@ class BenchmarkServiceImpl final : public BenchmarkService::Service {
           return Status(grpc::StatusCode::INTERNAL, "Error creating payload.");
         }
       }
-      stream->Write(response);
+      if (!stream->Write(response)) {
+        return Status(StatusCode::INTERNAL, "Server couldn't respond");
+      }
     }
     return Status::OK;
   }
diff --git a/test/cpp/server/server_builder_test.cc b/test/cpp/server/server_builder_test.cc
new file mode 100644
index 0000000000..1d9eda17b4
--- /dev/null
+++ b/test/cpp/server/server_builder_test.cc
@@ -0,0 +1,96 @@
+/*
+ *
+ * Copyright 2017, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc++/impl/codegen/config.h>
+#include <gtest/gtest.h>
+
+#include <grpc++/server.h>
+#include <grpc++/server_builder.h>
+
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
+#include "test/core/util/port.h"
+
+namespace grpc {
+namespace {
+
+testing::EchoTestService::Service g_service;
+
+grpc::string MakePort() {
+  std::ostringstream s;
+  int p = grpc_pick_unused_port_or_die();
+  s << "localhost:" << p;
+  return s.str();
+}
+
+grpc::string g_port = MakePort();
+
+TEST(ServerBuilderTest, NoOp) { ServerBuilder b; }
+
+TEST(ServerBuilderTest, CreateServerNoPorts) {
+  ServerBuilder().RegisterService(&g_service).BuildAndStart()->Shutdown();
+}
+
+TEST(ServerBuilderTest, CreateServerOnePort) {
+  ServerBuilder()
+      .RegisterService(&g_service)
+      .AddListeningPort(g_port, InsecureServerCredentials())
+      .BuildAndStart()
+      ->Shutdown();
+}
+
+TEST(ServerBuilderTest, CreateServerRepeatedPort) {
+  ServerBuilder()
+      .RegisterService(&g_service)
+      .AddListeningPort(g_port, InsecureServerCredentials())
+      .AddListeningPort(g_port, InsecureServerCredentials())
+      .BuildAndStart()
+      ->Shutdown();
+}
+
+TEST(ServerBuilderTest, CreateServerRepeatedPortWithDisallowedReusePort) {
+  EXPECT_EQ(ServerBuilder()
+                .RegisterService(&g_service)
+                .AddListeningPort(g_port, InsecureServerCredentials())
+                .AddListeningPort(g_port, InsecureServerCredentials())
+                .AddChannelArgument(GRPC_ARG_ALLOW_REUSEPORT, 0)
+                .BuildAndStart(),
+            nullptr);
+}
+
+}  // namespace
+}  // namespace grpc
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/test/http2_test/http2_server_health_check.py b/test/http2_test/http2_server_health_check.py
new file mode 100644
index 0000000000..dd9402b855
--- /dev/null
+++ b/test/http2_test/http2_server_health_check.py
@@ -0,0 +1,49 @@
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import hyper
+import sys
+
+# Utility to healthcheck the http2 server. Used when starting the server to
+# verify that the server is live before tests begin.
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--server_host', type=str, default='localhost')
+  parser.add_argument('--server_port', type=int, default=8080)
+  args = parser.parse_args()
+  server_host = args.server_host
+  server_port = args.server_port
+  conn = hyper.HTTP20Connection('%s:%d' % (server_host, server_port))
+  conn.request('POST', '/grpc.testing.TestService/UnaryCall')
+  resp = conn.get_response()
+  if resp.headers.get('grpc-encoding') is None:
+    sys.exit(1)
+  else:
+    sys.exit(0)
diff --git a/test/http2_test/http2_test_server.py b/test/http2_test/http2_test_server.py
index abde3433ad..46c3e00d18 100644
--- a/test/http2_test/http2_test_server.py
+++ b/test/http2_test/http2_test_server.py
@@ -31,6 +31,7 @@
 
 import argparse
 import logging
+import sys
 import twisted
 import twisted.internet
 import twisted.internet.endpoints
@@ -53,9 +54,11 @@ _TEST_CASE_MAPPING = {
   'max_streams': test_max_streams.TestcaseSettingsMaxStreams,
 }
 
+_exit_code = 0
+
 class H2Factory(twisted.internet.protocol.Factory):
   def __init__(self, testcase):
-    logging.info('Creating H2Factory for new connection.')
+    logging.info('Creating H2Factory for new connection (%s)', testcase)
     self._num_streams = 0
     self._testcase = testcase
 
@@ -83,6 +86,17 @@ def parse_arguments():
     )
   return parser.parse_args()
 
+def listen(endpoint, test_case):
+  deferred = endpoint.listen(H2Factory(test_case))
+  def listen_error(reason):
+    # If listening fails, we stop the reactor and exit the program
+    # with exit code 1.
+    global _exit_code
+    _exit_code = 1
+    logging.error('Listening failed: %s' % reason.value)
+    twisted.internet.reactor.stop()
+  deferred.addErrback(listen_error)
+
 def start_test_servers(base_port):
   """ Start one server per test case on incrementing port numbers
   beginning with base_port """
@@ -92,7 +106,9 @@ def start_test_servers(base_port):
     logging.warning('serving on port %d : %s'%(portnum, test_case))
     endpoint = twisted.internet.endpoints.TCP4ServerEndpoint(
       twisted.internet.reactor, portnum, backlog=128)
-    endpoint.listen(H2Factory(test_case))
+    # Wait until the reactor is running before calling endpoint.listen().
+    twisted.internet.reactor.callWhenRunning(listen, endpoint, test_case)
+
     index += 1
 
 if __name__ == '__main__':
@@ -102,3 +118,4 @@ if __name__ == '__main__':
   args = parse_arguments()
   start_test_servers(args.base_port)
   twisted.internet.reactor.run()
+  sys.exit(_exit_code)