Merge branch 'master' into corelimit2

65 files changed, 1991 insertions, 284 deletions

diff --git a/test/core/bad_client/bad_client.c b/test/core/bad_client/bad_client.c
index 832570a81d..1a2ca6f0c0 100644
--- a/test/core/bad_client/bad_client.c
+++ b/test/core/bad_client/bad_client.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -60,7 +60,7 @@ static void thd_func(void *arg) {
   gpr_event_set(&a->done_thd, (void *)1);
 }
 
-static void done_write(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void done_write(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   thd_args *a = arg;
   gpr_event_set(&a->done_write, (void *)1);
 }
diff --git a/test/core/bad_client/gen_build_yaml.py b/test/core/bad_client/gen_build_yaml.py
index a86a50065d..c538bffd71 100755
--- a/test/core/bad_client/gen_build_yaml.py
+++ b/test/core/bad_client/gen_build_yaml.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python2.7
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -35,15 +35,15 @@
 import collections
 import yaml
 
-TestOptions = collections.namedtuple('TestOptions', 'flaky')
-default_test_options = TestOptions(False)
+TestOptions = collections.namedtuple('TestOptions', 'flaky cpu_cost')
+default_test_options = TestOptions(False, 1.0)
 
 # maps test names to options
 BAD_CLIENT_TESTS = {
     'badreq': default_test_options,
-    'connection_prefix': default_test_options,
-    'headers': default_test_options,
-    'initial_settings_frame': default_test_options,
+    'connection_prefix': default_test_options._replace(cpu_cost=0.2),
+    'headers': default_test_options._replace(cpu_cost=0.2),
+    'initial_settings_frame': default_test_options._replace(cpu_cost=0.2),
     'server_registered_method': default_test_options,
     'simple_request': default_test_options,
     'window_overflow': default_test_options,
@@ -75,6 +75,7 @@ def main():
       'targets': [
           {
               'name': '%s_bad_client_test' % t,
+              'cpu_cost': BAD_CLIENT_TESTS[t].cpu_cost,
               'build': 'test',
               'language': 'c',
               'secure': 'no',
diff --git a/test/core/bad_ssl/gen_build_yaml.py b/test/core/bad_ssl/gen_build_yaml.py
index 15189d8b79..cc097a8fdf 100755
--- a/test/core/bad_ssl/gen_build_yaml.py
+++ b/test/core/bad_ssl/gen_build_yaml.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python2.7
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -35,13 +35,13 @@
 import collections
 import yaml
 
-TestOptions = collections.namedtuple('TestOptions', 'flaky')
-default_test_options = TestOptions(False)
+TestOptions = collections.namedtuple('TestOptions', 'flaky cpu_cost')
+default_test_options = TestOptions(False, 1.0)
 
 # maps test names to options
 BAD_CLIENT_TESTS = {
-  'cert': default_test_options,
-  'alpn': default_test_options,
+    'cert': default_test_options._replace(cpu_cost=0.1),
+    'alpn': default_test_options._replace(cpu_cost=0.1),
 }
 
 def main():
@@ -84,6 +84,7 @@ def main():
       for t in sorted(BAD_CLIENT_TESTS.keys())] + [
           {
               'name': 'bad_ssl_%s_test' % t,
+              'cpu_cost': BAD_CLIENT_TESTS[t].cpu_cost,
               'build': 'test',
               'language': 'c',
               'src': ['test/core/bad_ssl/bad_ssl_test.c'],
diff --git a/test/core/census/tag_set_test.c b/test/core/census/tag_set_test.c
new file mode 100644
index 0000000000..1056e98d55
--- /dev/null
+++ b/test/core/census/tag_set_test.c
@@ -0,0 +1,375 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+// Test census_tag_set functions, including encoding/decoding
+
+#include <grpc/census.h>
+#include <grpc/support/log.h>
+#include <grpc/support/time.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "test/core/util/test_config.h"
+
+static uint8_t one_byte_val = 7;
+static uint32_t four_byte_val = 0x12345678;
+static uint64_t eight_byte_val = 0x1234567890abcdef;
+
+// A set of tags Used to create a basic tag_set for testing. Each tag has a
+// unique set of flags. Note that replace_add_delete_test() relies on specific
+// offsets into this array - if you add or delete entries, you will also need
+// to change the test.
+#define BASIC_TAG_COUNT 8
+static census_tag basic_tags[BASIC_TAG_COUNT] = {
+    /* 0 */ {"key0", "printable", 10, 0},
+    /* 1 */ {"k1", "a", 2, CENSUS_TAG_PROPAGATE},
+    /* 2 */ {"k2", "longer printable string", 24, CENSUS_TAG_STATS},
+    /* 3 */ {"key_three", (char *)&one_byte_val, 1, CENSUS_TAG_BINARY},
+    /* 4 */ {"really_long_key_4", "random", 7,
+             CENSUS_TAG_PROPAGATE | CENSUS_TAG_STATS},
+    /* 5 */ {"k5", (char *)&four_byte_val, 4,
+             CENSUS_TAG_PROPAGATE | CENSUS_TAG_BINARY},
+    /* 6 */ {"k6", (char *)&eight_byte_val, 8,
+             CENSUS_TAG_STATS | CENSUS_TAG_BINARY},
+    /* 7 */ {"k7", (char *)&four_byte_val, 4,
+             CENSUS_TAG_PROPAGATE | CENSUS_TAG_STATS | CENSUS_TAG_BINARY}};
+
+// Set of tags used to modify the basic tag_set. Note that
+// replace_add_delete_test() relies on specific offsets into this array - if
+// you add or delete entries, you will also need to change the test. Other
+// tests that rely on specific instances have XXX_XXX_OFFSET definitions (also
+// change the defines below if you add/delete entires).
+#define MODIFY_TAG_COUNT 11
+static census_tag modify_tags[MODIFY_TAG_COUNT] = {
+#define REPLACE_VALUE_OFFSET 0
+    /* 0 */ {"key0", "replace printable", 18, 0},  // replaces tag value only
+#define ADD_TAG_OFFSET 1
+    /* 1 */ {"new_key", "xyzzy", 6, CENSUS_TAG_STATS},  // new tag
+#define DELETE_TAG_OFFSET 2
+    /* 2 */ {"k5", NULL, 5,
+             0},  // should delete tag, despite bogus value length
+    /* 3 */ {"k6", "foo", 0, 0},  // should delete tag, despite bogus value
+    /* 4 */ {"k6", "foo", 0, 0},  // try deleting already-deleted tag
+    /* 5 */ {"non-existent", NULL, 0, 0},  // another non-existent tag
+#define REPLACE_FLAG_OFFSET 6
+    /* 6 */ {"k1", "a", 2, 0},                   // change flags only
+    /* 7 */ {"k7", "bar", 4, CENSUS_TAG_STATS},  // change flags and value
+    /* 8 */ {"k2", (char *)&eight_byte_val, 8,
+             CENSUS_TAG_BINARY | CENSUS_TAG_PROPAGATE},  // more flags change
+                                                         // non-binary -> binary
+    /* 9 */ {"k6", "bar", 4, 0},  // add back tag, with different value
+    /* 10 */ {"foo", "bar", 4, CENSUS_TAG_PROPAGATE},  // another new tag
+};
+
+// Utility function to compare tags. Returns true if all fields match.
+static bool compare_tag(const census_tag *t1, const census_tag *t2) {
+  return (strcmp(t1->key, t2->key) == 0 && t1->value_len == t2->value_len &&
+          memcmp(t1->value, t2->value, t1->value_len) == 0 &&
+          t1->flags == t2->flags);
+}
+
+// Utility function to validate a tag exists in tag set.
+static bool validate_tag(const census_tag_set *cts, const census_tag *tag) {
+  census_tag tag2;
+  if (census_tag_set_get_tag_by_key(cts, tag->key, &tag2) != 1) return false;
+  return compare_tag(tag, &tag2);
+}
+
+// Create an empty tag_set.
+static void empty_test(void) {
+  struct census_tag_set *cts = census_tag_set_create(NULL, NULL, 0, NULL);
+  GPR_ASSERT(cts != NULL);
+  const census_tag_set_create_status *status =
+      census_tag_set_get_create_status(cts);
+  census_tag_set_create_status expected = {0, 0, 0, 0, 0, 0, 0, 0};
+  GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0);
+  census_tag_set_destroy(cts);
+}
+
+// Test create and iteration over basic tag set.
+static void basic_test(void) {
+  const census_tag_set_create_status *status;
+  struct census_tag_set *cts =
+      census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, &status);
+  census_tag_set_create_status expected = {2, 2, 4, 0, 8, 0, 0, 0};
+  GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0);
+  census_tag_set_iterator it;
+  census_tag_set_initialize_iterator(cts, &it);
+  census_tag tag;
+  while (census_tag_set_next_tag(&it, &tag)) {
+    // can't rely on tag return order: make sure it matches exactly one.
+    int matches = 0;
+    for (int i = 0; i < BASIC_TAG_COUNT; i++) {
+      if (compare_tag(&tag, &basic_tags[i])) matches++;
+    }
+    GPR_ASSERT(matches == 1);
+  }
+  census_tag_set_destroy(cts);
+}
+
+// Test that census_tag_set_get_tag_by_key().
+static void lookup_by_key_test(void) {
+  struct census_tag_set *cts =
+      census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL);
+  census_tag tag;
+  for (int i = 0; i < BASIC_TAG_COUNT; i++) {
+    GPR_ASSERT(census_tag_set_get_tag_by_key(cts, basic_tags[i].key, &tag) ==
+               1);
+    GPR_ASSERT(compare_tag(&tag, &basic_tags[i]));
+  }
+  // non-existent keys
+  GPR_ASSERT(census_tag_set_get_tag_by_key(cts, "key", &tag) == 0);
+  GPR_ASSERT(census_tag_set_get_tag_by_key(cts, "key01", &tag) == 0);
+  GPR_ASSERT(census_tag_set_get_tag_by_key(cts, "k9", &tag) == 0);
+  GPR_ASSERT(census_tag_set_get_tag_by_key(cts, "random", &tag) == 0);
+  GPR_ASSERT(census_tag_set_get_tag_by_key(cts, "", &tag) == 0);
+  census_tag_set_destroy(cts);
+}
+
+// Try creating tag set with invalid entries.
+static void invalid_test(void) {
+  char key[300];
+  memset(key, 'k', 299);
+  key[299] = 0;
+  char value[300];
+  memset(value, 'v', 300);
+  census_tag tag = {key, value, 3, CENSUS_TAG_BINARY};
+  // long keys, short value. Key lengths (including terminator) should be
+  // <= 255 (CENSUS_MAX_TAG_KV_LEN)
+  GPR_ASSERT(strlen(key) == 299);
+  const census_tag_set_create_status *status;
+  struct census_tag_set *cts = census_tag_set_create(NULL, &tag, 1, &status);
+  census_tag_set_create_status expected = {0, 0, 0, 0, 0, 0, 1, 0};
+  GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0);
+  census_tag_set_destroy(cts);
+  key[CENSUS_MAX_TAG_KV_LEN] = 0;
+  GPR_ASSERT(strlen(key) == CENSUS_MAX_TAG_KV_LEN);
+  cts = census_tag_set_create(NULL, &tag, 1, &status);
+  GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0);
+  census_tag_set_destroy(cts);
+  key[CENSUS_MAX_TAG_KV_LEN - 1] = 0;
+  GPR_ASSERT(strlen(key) == CENSUS_MAX_TAG_KV_LEN - 1);
+  cts = census_tag_set_create(NULL, &tag, 1, &status);
+  census_tag_set_create_status expected2 = {0, 0, 1, 0, 1, 0, 0, 0};
+  GPR_ASSERT(memcmp(status, &expected2, sizeof(expected2)) == 0);
+  census_tag_set_destroy(cts);
+  // now try with long values
+  tag.value_len = 300;
+  cts = census_tag_set_create(NULL, &tag, 1, &status);
+  GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0);
+  census_tag_set_destroy(cts);
+  tag.value_len = CENSUS_MAX_TAG_KV_LEN + 1;
+  cts = census_tag_set_create(NULL, &tag, 1, &status);
+  GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0);
+  census_tag_set_destroy(cts);
+  tag.value_len = CENSUS_MAX_TAG_KV_LEN;
+  cts = census_tag_set_create(NULL, &tag, 1, &status);
+  GPR_ASSERT(memcmp(status, &expected2, sizeof(expected2)) == 0);
+  census_tag_set_destroy(cts);
+  // 0 length key.
+  key[0] = 0;
+  cts = census_tag_set_create(NULL, &tag, 1, &status);
+  GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0);
+  census_tag_set_destroy(cts);
+}
+
+// Make a copy of a tag set
+static void copy_test(void) {
+  struct census_tag_set *cts =
+      census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL);
+  const census_tag_set_create_status *status;
+  struct census_tag_set *cts2 = census_tag_set_create(cts, NULL, 0, &status);
+  census_tag_set_create_status expected = {2, 2, 4, 0, 0, 0, 0, 0};
+  GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0);
+  for (int i = 0; i < BASIC_TAG_COUNT; i++) {
+    census_tag tag;
+    GPR_ASSERT(census_tag_set_get_tag_by_key(cts2, basic_tags[i].key, &tag) ==
+               1);
+    GPR_ASSERT(compare_tag(&tag, &basic_tags[i]));
+  }
+  census_tag_set_destroy(cts);
+  census_tag_set_destroy(cts2);
+}
+
+// replace a single tag value
+static void replace_value_test(void) {
+  struct census_tag_set *cts =
+      census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL);
+  const census_tag_set_create_status *status;
+  struct census_tag_set *cts2 = census_tag_set_create(
+      cts, modify_tags + REPLACE_VALUE_OFFSET, 1, &status);
+  census_tag_set_create_status expected = {2, 2, 4, 0, 0, 1, 0, 0};
+  GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0);
+  census_tag tag;
+  GPR_ASSERT(census_tag_set_get_tag_by_key(
+                 cts2, modify_tags[REPLACE_VALUE_OFFSET].key, &tag) == 1);
+  GPR_ASSERT(compare_tag(&tag, &modify_tags[REPLACE_VALUE_OFFSET]));
+  census_tag_set_destroy(cts);
+  census_tag_set_destroy(cts2);
+}
+
+// replace a single tags flags
+static void replace_flags_test(void) {
+  struct census_tag_set *cts =
+      census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL);
+  const census_tag_set_create_status *status;
+  struct census_tag_set *cts2 =
+      census_tag_set_create(cts, modify_tags + REPLACE_FLAG_OFFSET, 1, &status);
+  census_tag_set_create_status expected = {1, 2, 5, 0, 0, 1, 0, 0};
+  GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0);
+  census_tag tag;
+  GPR_ASSERT(census_tag_set_get_tag_by_key(
+                 cts2, modify_tags[REPLACE_FLAG_OFFSET].key, &tag) == 1);
+  GPR_ASSERT(compare_tag(&tag, &modify_tags[REPLACE_FLAG_OFFSET]));
+  census_tag_set_destroy(cts);
+  census_tag_set_destroy(cts2);
+}
+
+// delete a single tag.
+static void delete_tag_test(void) {
+  struct census_tag_set *cts =
+      census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL);
+  const census_tag_set_create_status *status;
+  struct census_tag_set *cts2 =
+      census_tag_set_create(cts, modify_tags + DELETE_TAG_OFFSET, 1, &status);
+  census_tag_set_create_status expected = {2, 1, 4, 1, 0, 0, 0, 0};
+  GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0);
+  census_tag tag;
+  GPR_ASSERT(census_tag_set_get_tag_by_key(
+                 cts2, modify_tags[DELETE_TAG_OFFSET].key, &tag) == 0);
+  census_tag_set_destroy(cts);
+  census_tag_set_destroy(cts2);
+}
+
+// add a single new tag.
+static void add_tag_test(void) {
+  struct census_tag_set *cts =
+      census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL);
+  const census_tag_set_create_status *status;
+  struct census_tag_set *cts2 =
+      census_tag_set_create(cts, modify_tags + ADD_TAG_OFFSET, 1, &status);
+  census_tag_set_create_status expected = {2, 2, 5, 0, 1, 0, 0, 0};
+  GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0);
+  census_tag tag;
+  GPR_ASSERT(census_tag_set_get_tag_by_key(
+                 cts2, modify_tags[ADD_TAG_OFFSET].key, &tag) == 1);
+  GPR_ASSERT(compare_tag(&tag, &modify_tags[ADD_TAG_OFFSET]));
+  census_tag_set_destroy(cts);
+  census_tag_set_destroy(cts2);
+}
+
+// test many changes at once.
+static void replace_add_delete_test(void) {
+  struct census_tag_set *cts =
+      census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL);
+  const census_tag_set_create_status *status;
+  struct census_tag_set *cts2 =
+      census_tag_set_create(cts, modify_tags, MODIFY_TAG_COUNT, &status);
+  census_tag_set_create_status expected = {2, 1, 6, 2, 3, 4, 0, 2};
+  GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0);
+  // validate tag set contents. Use specific indices into the two arrays
+  // holding tag values.
+  GPR_ASSERT(validate_tag(cts2, &basic_tags[3]));
+  GPR_ASSERT(validate_tag(cts2, &basic_tags[4]));
+  GPR_ASSERT(validate_tag(cts2, &modify_tags[0]));
+  GPR_ASSERT(validate_tag(cts2, &modify_tags[1]));
+  GPR_ASSERT(validate_tag(cts2, &modify_tags[6]));
+  GPR_ASSERT(validate_tag(cts2, &modify_tags[7]));
+  GPR_ASSERT(validate_tag(cts2, &modify_tags[8]));
+  GPR_ASSERT(validate_tag(cts2, &modify_tags[9]));
+  GPR_ASSERT(validate_tag(cts2, &modify_tags[10]));
+  GPR_ASSERT(!validate_tag(cts2, &basic_tags[0]));
+  GPR_ASSERT(!validate_tag(cts2, &basic_tags[1]));
+  GPR_ASSERT(!validate_tag(cts2, &basic_tags[2]));
+  GPR_ASSERT(!validate_tag(cts2, &basic_tags[5]));
+  GPR_ASSERT(!validate_tag(cts2, &basic_tags[6]));
+  GPR_ASSERT(!validate_tag(cts2, &basic_tags[7]));
+  census_tag_set_destroy(cts);
+  census_tag_set_destroy(cts2);
+}
+
+#define BUF_SIZE 200
+
+// test encode/decode.
+static void encode_decode_test(void) {
+  char buffer[BUF_SIZE];
+  struct census_tag_set *cts =
+      census_tag_set_create(NULL, basic_tags, BASIC_TAG_COUNT, NULL);
+  size_t print_bsize;
+  size_t bin_bsize;
+  // Test with too small a buffer
+  GPR_ASSERT(census_tag_set_encode(cts, buffer, 2, &print_bsize, &bin_bsize) ==
+             NULL);
+  char *b_buffer =
+      census_tag_set_encode(cts, buffer, BUF_SIZE, &print_bsize, &bin_bsize);
+  GPR_ASSERT(b_buffer != NULL && print_bsize > 0 && bin_bsize > 0 &&
+             print_bsize + bin_bsize <= BUF_SIZE &&
+             b_buffer == buffer + print_bsize);
+  census_tag_set *cts2 =
+      census_tag_set_decode(buffer, print_bsize, b_buffer, bin_bsize);
+  GPR_ASSERT(cts2 != NULL);
+  const census_tag_set_create_status *status =
+      census_tag_set_get_create_status(cts2);
+  census_tag_set_create_status expected = {2, 2, 0, 0, 0, 0, 0, 0};
+  GPR_ASSERT(memcmp(status, &expected, sizeof(expected)) == 0);
+  for (int i = 0; i < BASIC_TAG_COUNT; i++) {
+    census_tag tag;
+    if (CENSUS_TAG_IS_PROPAGATED(basic_tags[i].flags)) {
+      GPR_ASSERT(census_tag_set_get_tag_by_key(cts2, basic_tags[i].key, &tag) ==
+                 1);
+      GPR_ASSERT(compare_tag(&tag, &basic_tags[i]));
+    } else {
+      GPR_ASSERT(census_tag_set_get_tag_by_key(cts2, basic_tags[i].key, &tag) ==
+                 0);
+    }
+  }
+  census_tag_set_destroy(cts2);
+  census_tag_set_destroy(cts);
+}
+
+int main(int argc, char *argv[]) {
+  grpc_test_init(argc, argv);
+  empty_test();
+  basic_test();
+  lookup_by_key_test();
+  invalid_test();
+  copy_test();
+  replace_value_test();
+  replace_flags_test();
+  delete_tag_test();
+  add_tag_test();
+  replace_add_delete_test();
+  encode_decode_test();
+  return 0;
+}
diff --git a/test/core/channel/channel_stack_test.c b/test/core/channel/channel_stack_test.c
index f1bb37c0bf..e19e9a57ae 100644
--- a/test/core/channel/channel_stack_test.c
+++ b/test/core/channel/channel_stack_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -81,12 +81,12 @@ static char *get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
   return gpr_strdup("peer");
 }
 
-static void free_channel(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void free_channel(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   grpc_channel_stack_destroy(exec_ctx, arg);
   gpr_free(arg);
 }
 
-static void free_call(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void free_call(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   grpc_call_stack_destroy(exec_ctx, arg);
   gpr_free(arg);
 }
diff --git a/test/core/client_config/set_initial_connect_string_test.c b/test/core/client_config/set_initial_connect_string_test.c
index ceca56c833..bcd1f26123 100644
--- a/test/core/client_config/set_initial_connect_string_test.c
+++ b/test/core/client_config/set_initial_connect_string_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -64,7 +64,7 @@ static int server_port;
 static struct rpc_state state;
 static grpc_closure on_read;
 
-static void handle_read(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void handle_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   GPR_ASSERT(success);
   gpr_slice_buffer_move_into(&state.temp_incoming_buffer,
                              &state.incoming_buffer);
@@ -78,7 +78,8 @@ static void handle_read(grpc_exec_ctx *exec_ctx, void *arg, int success) {
   }
 }
 
-static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp) {
+static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp,
+                       grpc_tcp_server_acceptor *acceptor) {
   test_tcp_server *server = arg;
   grpc_closure_init(&on_read, handle_read, NULL);
   gpr_slice_buffer_init(&state.incoming_buffer);
diff --git a/test/core/end2end/fixtures/h2_uchannel.c b/test/core/end2end/fixtures/h2_uchannel.c
index 9b622e80d6..5ab64f9800 100644
--- a/test/core/end2end/fixtures/h2_uchannel.c
+++ b/test/core/end2end/fixtures/h2_uchannel.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -81,7 +81,7 @@ static void connector_unref(grpc_exec_ctx *exec_ctx, grpc_connector *con) {
   }
 }
 
-static void connected(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void connected(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   connector *c = arg;
   grpc_closure *notify;
   grpc_endpoint *tcp = c->tcp;
@@ -240,7 +240,7 @@ static grpc_end2end_test_fixture chttp2_create_fixture_micro_fullstack(
 grpc_connectivity_state g_state = GRPC_CHANNEL_IDLE;
 grpc_pollset_set g_interested_parties;
 
-static void state_changed(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void state_changed(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   if (g_state != GRPC_CHANNEL_READY) {
     grpc_subchannel_notify_on_state_change(
         exec_ctx, arg, &g_interested_parties, &g_state,
@@ -248,7 +248,7 @@ static void state_changed(grpc_exec_ctx *exec_ctx, void *arg, int success) {
   }
 }
 
-static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   grpc_pollset_destroy(arg);
 }
 
diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py
index 932ef2fd96..f24dbe72cf 100755
--- a/test/core/end2end/gen_build_yaml.py
+++ b/test/core/end2end/gen_build_yaml.py
@@ -77,40 +77,42 @@ END2END_FIXTURES = {
 }
 
 TestOptions = collections.namedtuple(
-    'TestOptions', 'needs_fullstack needs_dns proxyable secure traceable')
-default_test_options = TestOptions(False, False, True, False, True)
+    'TestOptions', 'needs_fullstack needs_dns proxyable secure traceable cpu_cost')
+default_test_options = TestOptions(False, False, True, False, True, 1.0)
 connectivity_test_options = default_test_options._replace(needs_fullstack=True)
 
+LOWCPU = 0.1
+
 # maps test names to options
 END2END_TESTS = {
     'bad_hostname': default_test_options,
     'binary_metadata': default_test_options,
     'call_creds': default_test_options._replace(secure=True),
-    'cancel_after_accept': default_test_options,
-    'cancel_after_client_done': default_test_options,
-    'cancel_after_invoke': default_test_options,
-    'cancel_before_invoke': default_test_options,
-    'cancel_in_a_vacuum': default_test_options,
-    'cancel_with_status': default_test_options,
-    'channel_connectivity': connectivity_test_options._replace(proxyable=False),
+    'cancel_after_accept': default_test_options._replace(cpu_cost=LOWCPU),
+    'cancel_after_client_done': default_test_options._replace(cpu_cost=LOWCPU),
+    'cancel_after_invoke': default_test_options._replace(cpu_cost=LOWCPU),
+    'cancel_before_invoke': default_test_options._replace(cpu_cost=LOWCPU),
+    'cancel_in_a_vacuum': default_test_options._replace(cpu_cost=LOWCPU),
+    'cancel_with_status': default_test_options._replace(cpu_cost=LOWCPU),
+    'channel_connectivity': connectivity_test_options._replace(proxyable=False, cpu_cost=LOWCPU),
     'channel_ping': connectivity_test_options._replace(proxyable=False),
-    'compressed_payload': default_test_options._replace(proxyable=False),
+    'compressed_payload': default_test_options._replace(proxyable=False, cpu_cost=LOWCPU),
     'default_host': default_test_options._replace(needs_fullstack=True,
                                                   needs_dns=True),
     'disappearing_server': connectivity_test_options,
     'empty_batch': default_test_options,
-    'graceful_server_shutdown': default_test_options,
+    'graceful_server_shutdown': default_test_options._replace(cpu_cost=LOWCPU),
     'hpack_size': default_test_options._replace(proxyable=False,
                                                 traceable=False),
     'high_initial_seqno': default_test_options,
     'invoke_large_request': default_test_options,
     'large_metadata': default_test_options,
     'max_concurrent_streams': default_test_options._replace(proxyable=False),
-    'max_message_length': default_test_options,
+    'max_message_length': default_test_options._replace(cpu_cost=LOWCPU),
     'metadata': default_test_options,
     'negative_deadline': default_test_options,
     'no_op': default_test_options,
-    'payload': default_test_options,
+    'payload': default_test_options._replace(cpu_cost=LOWCPU),
     'ping_pong_streaming': default_test_options,
     'registered_call': default_test_options,
     'request_with_flags': default_test_options._replace(proxyable=False),
@@ -118,7 +120,7 @@ END2END_TESTS = {
     'server_finishes_request': default_test_options,
     'shutdown_finishes_calls': default_test_options,
     'shutdown_finishes_tags': default_test_options,
-    'simple_delayed_request': connectivity_test_options,
+    'simple_delayed_request': connectivity_test_options._replace(cpu_cost=LOWCPU),
     'simple_request': default_test_options,
     'trailing_metadata': default_test_options,
 }
@@ -252,6 +254,7 @@ def main():
                                    END2END_FIXTURES[f].platforms, 'mac')),
               'flaky': False,
               'language': 'c',
+              'cpu_cost': END2END_TESTS[t].cpu_cost,
           }
           for f in sorted(END2END_FIXTURES.keys())
           for t in sorted(END2END_TESTS.keys()) if compatible(f, t)
@@ -266,6 +269,7 @@ def main():
                                    END2END_FIXTURES[f].platforms, 'mac')),
               'flaky': False,
               'language': 'c',
+              'cpu_cost': END2END_TESTS[t].cpu_cost,
           }
           for f in sorted(END2END_FIXTURES.keys())
           if not END2END_FIXTURES[f].secure
diff --git a/test/core/fling/client.c b/test/core/fling/client.c
index 99b30d6c4a..02db681cfd 100644
--- a/test/core/fling/client.c
+++ b/test/core/fling/client.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -201,13 +201,16 @@ int main(int argc, char **argv) {
 
   sc.init();
 
-  for (i = 0; i < 1000; i++) {
+  gpr_timespec end_warmup = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(3);
+  gpr_timespec end_profiling = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(30);
+
+  while (gpr_time_cmp(gpr_now(end_warmup.clock_type), end_warmup) < 0) {
     sc.do_one_step();
   }
 
   gpr_log(GPR_INFO, "start profiling");
   grpc_profiler_start("client.prof");
-  for (i = 0; i < 100000; i++) {
+  while (gpr_time_cmp(gpr_now(end_profiling.clock_type), end_profiling) < 0) {
     start = now();
     sc.do_one_step();
     stop = now();
diff --git a/test/core/httpcli/httpcli_test.c b/test/core/httpcli/httpcli_test.c
index 612388c61d..651ef1fa3b 100644
--- a/test/core/httpcli/httpcli_test.c
+++ b/test/core/httpcli/httpcli_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -131,7 +131,7 @@ static void test_post(int port) {
   gpr_free(host);
 }
 
-static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, int success) {
+static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool success) {
   grpc_pollset_destroy(p);
 }
 
diff --git a/test/core/httpcli/httpscli_test.c b/test/core/httpcli/httpscli_test.c
index ba5660bd18..db41be17e7 100644
--- a/test/core/httpcli/httpscli_test.c
+++ b/test/core/httpcli/httpscli_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -133,7 +133,7 @@ static void test_post(int port) {
   gpr_free(host);
 }
 
-static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, int success) {
+static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool success) {
   grpc_pollset_destroy(p);
 }
 
diff --git a/test/core/iomgr/endpoint_pair_test.c b/test/core/iomgr/endpoint_pair_test.c
index ff590cf2d5..7e266ebfb9 100644
--- a/test/core/iomgr/endpoint_pair_test.c
+++ b/test/core/iomgr/endpoint_pair_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -65,7 +65,7 @@ static grpc_endpoint_test_config configs[] = {
     {"tcp/tcp_socketpair", create_fixture_endpoint_pair, clean_up},
 };
 
-static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, int success) {
+static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool success) {
   grpc_pollset_destroy(p);
 }
 
diff --git a/test/core/iomgr/endpoint_tests.c b/test/core/iomgr/endpoint_tests.c
index a66fe32fde..1b6a78da9a 100644
--- a/test/core/iomgr/endpoint_tests.c
+++ b/test/core/iomgr/endpoint_tests.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -127,7 +127,7 @@ struct read_and_write_test_state {
 };
 
 static void read_and_write_test_read_handler(grpc_exec_ctx *exec_ctx,
-                                             void *data, int success) {
+                                             void *data, bool success) {
   struct read_and_write_test_state *state = data;
 
   state->bytes_read += count_slices(
@@ -145,7 +145,7 @@ static void read_and_write_test_read_handler(grpc_exec_ctx *exec_ctx,
 }
 
 static void read_and_write_test_write_handler(grpc_exec_ctx *exec_ctx,
-                                              void *data, int success) {
+                                              void *data, bool success) {
   struct read_and_write_test_state *state = data;
   gpr_slice *slices = NULL;
   size_t nslices;
diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c
index b186520729..347a86af10 100644
--- a/test/core/iomgr/fd_posix_test.c
+++ b/test/core/iomgr/fd_posix_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -118,7 +118,7 @@ typedef struct {
 /* Called when an upload session can be safely shutdown.
    Close session FD and start to shutdown listen FD. */
 static void session_shutdown_cb(grpc_exec_ctx *exec_ctx, void *arg, /*session */
-                                int success) {
+                                bool success) {
   session *se = arg;
   server *sv = se->sv;
   grpc_fd_orphan(exec_ctx, se->em_fd, NULL, NULL, "a");
@@ -129,7 +129,7 @@ static void session_shutdown_cb(grpc_exec_ctx *exec_ctx, void *arg, /*session */
 
 /* Called when data become readable in a session. */
 static void session_read_cb(grpc_exec_ctx *exec_ctx, void *arg, /*session */
-                            int success) {
+                            bool success) {
   session *se = arg;
   int fd = se->em_fd->fd;
 
@@ -187,7 +187,7 @@ static void listen_shutdown_cb(grpc_exec_ctx *exec_ctx, void *arg /*server */,
 
 /* Called when a new TCP connection request arrives in the listening port. */
 static void listen_cb(grpc_exec_ctx *exec_ctx, void *arg, /*=sv_arg*/
-                      int success) {
+                      bool success) {
   server *sv = arg;
   int fd;
   int flags;
@@ -301,7 +301,7 @@ static void client_session_shutdown_cb(grpc_exec_ctx *exec_ctx,
 
 /* Write as much as possible, then register notify_on_write. */
 static void client_session_write(grpc_exec_ctx *exec_ctx, void *arg, /*client */
-                                 int success) {
+                                 bool success) {
   client *cl = arg;
   int fd = cl->em_fd->fd;
   ssize_t write_once = 0;
@@ -399,7 +399,7 @@ static void test_grpc_fd(void) {
 }
 
 typedef struct fd_change_data {
-  void (*cb_that_ran)(grpc_exec_ctx *exec_ctx, void *, int success);
+  grpc_iomgr_cb_func cb_that_ran;
 } fd_change_data;
 
 void init_change_data(fd_change_data *fdc) { fdc->cb_that_ran = NULL; }
@@ -407,7 +407,7 @@ void init_change_data(fd_change_data *fdc) { fdc->cb_that_ran = NULL; }
 void destroy_change_data(fd_change_data *fdc) {}
 
 static void first_read_callback(grpc_exec_ctx *exec_ctx,
-                                void *arg /* fd_change_data */, int success) {
+                                void *arg /* fd_change_data */, bool success) {
   fd_change_data *fdc = arg;
 
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
@@ -417,7 +417,7 @@ static void first_read_callback(grpc_exec_ctx *exec_ctx,
 }
 
 static void second_read_callback(grpc_exec_ctx *exec_ctx,
-                                 void *arg /* fd_change_data */, int success) {
+                                 void *arg /* fd_change_data */, bool success) {
   fd_change_data *fdc = arg;
 
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
@@ -510,7 +510,7 @@ static void test_grpc_fd_change(void) {
   close(sv[1]);
 }
 
-static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, int success) {
+static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool success) {
   grpc_pollset_destroy(p);
 }
 
diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c
index 833ceace54..9725d8a3b6 100644
--- a/test/core/iomgr/tcp_client_posix_test.c
+++ b/test/core/iomgr/tcp_client_posix_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -63,7 +63,7 @@ static void finish_connection() {
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
 }
 
-static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   GPR_ASSERT(g_connecting != NULL);
   GPR_ASSERT(success);
   grpc_endpoint_shutdown(exec_ctx, g_connecting);
@@ -72,7 +72,7 @@ static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, int success) {
   finish_connection();
 }
 
-static void must_fail(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void must_fail(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   GPR_ASSERT(g_connecting == NULL);
   GPR_ASSERT(!success);
   finish_connection();
@@ -258,7 +258,7 @@ void test_times_out(void) {
   }
 }
 
-static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, int success) {
+static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool success) {
   grpc_pollset_destroy(p);
 }
 
diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c
index e0136b3cd7..d290c6bc3a 100644
--- a/test/core/iomgr/tcp_posix_test.c
+++ b/test/core/iomgr/tcp_posix_test.c
@@ -138,7 +138,7 @@ static size_t count_slices(gpr_slice *slices, size_t nslices,
   return num_bytes;
 }
 
-static void read_cb(grpc_exec_ctx *exec_ctx, void *user_data, int success) {
+static void read_cb(grpc_exec_ctx *exec_ctx, void *user_data, bool success) {
   struct read_socket_state *state = (struct read_socket_state *)user_data;
   size_t read_bytes;
   int current_data;
@@ -280,7 +280,7 @@ static gpr_slice *allocate_blocks(size_t num_bytes, size_t slice_size,
 }
 
 static void write_done(grpc_exec_ctx *exec_ctx,
-                       void *user_data /* write_socket_state */, int success) {
+                       void *user_data /* write_socket_state */, bool success) {
   struct write_socket_state *state = (struct write_socket_state *)user_data;
   gpr_log(GPR_INFO, "Write done callback called");
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
@@ -383,7 +383,7 @@ static void write_test(size_t num_bytes, size_t slice_size) {
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-void on_fd_released(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+void on_fd_released(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   int *done = arg;
   *done = 1;
   grpc_pollset_kick(&g_pollset, NULL);
@@ -503,7 +503,7 @@ static grpc_endpoint_test_config configs[] = {
     {"tcp/tcp_socketpair", create_fixture_tcp_socketpair, clean_up},
 };
 
-static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, int success) {
+static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool success) {
   grpc_pollset_destroy(p);
 }
 
diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c
index 530381e37f..272d97bfcb 100644
--- a/test/core/iomgr/tcp_server_posix_test.c
+++ b/test/core/iomgr/tcp_server_posix_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -33,11 +33,15 @@
 
 #include "src/core/iomgr/tcp_server.h"
 #include "src/core/iomgr/iomgr.h"
+#include "src/core/iomgr/sockaddr_utils.h"
+#include <grpc/grpc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/time.h>
+#include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
 
+#include <errno.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <string.h>
@@ -48,11 +52,69 @@
 static grpc_pollset g_pollset;
 static int g_nconnects = 0;
 
-static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp) {
+typedef struct on_connect_result {
+  /* Owns a ref to server. */
+  grpc_tcp_server *server;
+  unsigned port_index;
+  unsigned fd_index;
+  int server_fd;
+} on_connect_result;
+
+typedef struct server_weak_ref {
+  grpc_tcp_server *server;
+
+  /* arg is this server_weak_ref. */
+  grpc_closure server_shutdown;
+} server_weak_ref;
+
+static on_connect_result g_result = {NULL, 0, 0, -1};
+
+static void on_connect_result_init(on_connect_result *result) {
+  result->server = NULL;
+  result->port_index = 0;
+  result->fd_index = 0;
+  result->server_fd = -1;
+}
+
+static void on_connect_result_set(on_connect_result *result,
+                                  const grpc_tcp_server_acceptor *acceptor) {
+  result->server = grpc_tcp_server_ref(acceptor->from_server);
+  result->port_index = acceptor->port_index;
+  result->fd_index = acceptor->fd_index;
+  result->server_fd = grpc_tcp_server_port_fd(
+      result->server, acceptor->port_index, acceptor->fd_index);
+}
+
+static void server_weak_ref_shutdown(grpc_exec_ctx *exec_ctx, void *arg,
+                                     bool success) {
+  server_weak_ref *weak_ref = arg;
+  weak_ref->server = NULL;
+}
+
+static void server_weak_ref_init(server_weak_ref *weak_ref) {
+  weak_ref->server = NULL;
+  grpc_closure_init(&weak_ref->server_shutdown, server_weak_ref_shutdown,
+                    weak_ref);
+}
+
+/* Make weak_ref->server_shutdown a shutdown_starting cb on server.
+   grpc_tcp_server promises that the server object will live until
+   weak_ref->server_shutdown has returned. A strong ref on grpc_tcp_server
+   should be held until server_weak_ref_set() returns to avoid a race where the
+   server is deleted before the shutdown_starting cb is added. */
+static void server_weak_ref_set(server_weak_ref *weak_ref,
+                                grpc_tcp_server *server) {
+  grpc_tcp_server_shutdown_starting_add(server, &weak_ref->server_shutdown);
+  weak_ref->server = server;
+}
+
+static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp,
+                       grpc_tcp_server_acceptor *acceptor) {
   grpc_endpoint_shutdown(exec_ctx, tcp);
   grpc_endpoint_destroy(exec_ctx, tcp);
 
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
+  on_connect_result_set(&g_result, acceptor);
   g_nconnects++;
   grpc_pollset_kick(&g_pollset, NULL);
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
@@ -60,111 +122,188 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp) {
 
 static void test_no_op(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_tcp_server *s = grpc_tcp_server_create();
-  grpc_tcp_server_destroy(&exec_ctx, s, NULL);
+  grpc_tcp_server *s = grpc_tcp_server_create(NULL);
+  grpc_tcp_server_unref(&exec_ctx, s);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static void test_no_op_with_start(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_tcp_server *s = grpc_tcp_server_create();
+  grpc_tcp_server *s = grpc_tcp_server_create(NULL);
   LOG_TEST("test_no_op_with_start");
   grpc_tcp_server_start(&exec_ctx, s, NULL, 0, on_connect, NULL);
-  grpc_tcp_server_destroy(&exec_ctx, s, NULL);
+  grpc_tcp_server_unref(&exec_ctx, s);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static void test_no_op_with_port(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   struct sockaddr_in addr;
-  grpc_tcp_server *s = grpc_tcp_server_create();
+  grpc_tcp_server *s = grpc_tcp_server_create(NULL);
   LOG_TEST("test_no_op_with_port");
 
   memset(&addr, 0, sizeof(addr));
   addr.sin_family = AF_INET;
   GPR_ASSERT(
-      grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, sizeof(addr)));
+      grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, sizeof(addr)) > 0);
 
-  grpc_tcp_server_destroy(&exec_ctx, s, NULL);
+  grpc_tcp_server_unref(&exec_ctx, s);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
 static void test_no_op_with_port_and_start(void) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   struct sockaddr_in addr;
-  grpc_tcp_server *s = grpc_tcp_server_create();
+  grpc_tcp_server *s = grpc_tcp_server_create(NULL);
   LOG_TEST("test_no_op_with_port_and_start");
 
   memset(&addr, 0, sizeof(addr));
   addr.sin_family = AF_INET;
   GPR_ASSERT(
-      grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, sizeof(addr)));
+      grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, sizeof(addr)) > 0);
 
   grpc_tcp_server_start(&exec_ctx, s, NULL, 0, on_connect, NULL);
 
-  grpc_tcp_server_destroy(&exec_ctx, s, NULL);
+  grpc_tcp_server_unref(&exec_ctx, s);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void test_connect(int n) {
+static void tcp_connect(grpc_exec_ctx *exec_ctx, const struct sockaddr *remote,
+                        socklen_t remote_len, on_connect_result *result) {
+  gpr_timespec deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10);
+  int clifd = socket(remote->sa_family, SOCK_STREAM, 0);
+  int nconnects_before;
+
+  gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
+  nconnects_before = g_nconnects;
+  on_connect_result_init(&g_result);
+  GPR_ASSERT(clifd >= 0);
+  gpr_log(GPR_DEBUG, "start connect");
+  GPR_ASSERT(connect(clifd, remote, remote_len) == 0);
+  gpr_log(GPR_DEBUG, "wait");
+  while (g_nconnects == nconnects_before &&
+         gpr_time_cmp(deadline, gpr_now(deadline.clock_type)) > 0) {
+    grpc_pollset_worker worker;
+    grpc_pollset_work(exec_ctx, &g_pollset, &worker,
+                      gpr_now(GPR_CLOCK_MONOTONIC), deadline);
+    gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
+    grpc_exec_ctx_finish(exec_ctx);
+    gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
+  }
+  gpr_log(GPR_DEBUG, "wait done");
+  GPR_ASSERT(g_nconnects == nconnects_before + 1);
+  close(clifd);
+  *result = g_result;
+
+  gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
+}
+
+/* Tests a tcp server with multiple ports. TODO(daniel-j-born): Multiple fds for
+   the same port should be tested. */
+static void test_connect(unsigned n) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   struct sockaddr_storage addr;
+  struct sockaddr_storage addr1;
   socklen_t addr_len = sizeof(addr);
-  int svrfd, clifd;
-  grpc_tcp_server *s = grpc_tcp_server_create();
-  int nconnects_before;
-  gpr_timespec deadline;
+  unsigned svr_fd_count;
+  int svr_port;
+  unsigned svr1_fd_count;
+  int svr1_port;
+  grpc_tcp_server *s = grpc_tcp_server_create(NULL);
   grpc_pollset *pollsets[1];
-  int i;
+  unsigned i;
+  server_weak_ref weak_ref;
+  server_weak_ref_init(&weak_ref);
   LOG_TEST("test_connect");
   gpr_log(GPR_INFO, "clients=%d", n);
-
   memset(&addr, 0, sizeof(addr));
-  addr.ss_family = AF_INET;
-  GPR_ASSERT(grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, addr_len));
+  memset(&addr1, 0, sizeof(addr1));
+  addr.ss_family = addr1.ss_family = AF_INET;
+  svr_port = grpc_tcp_server_add_port(s, (struct sockaddr *)&addr, addr_len);
+  GPR_ASSERT(svr_port > 0);
+  /* Cannot use wildcard (port==0), because add_port() will try to reuse the
+     same port as a previous add_port(). */
+  svr1_port = grpc_pick_unused_port_or_die();
+  grpc_sockaddr_set_port((struct sockaddr *)&addr1, svr1_port);
+  GPR_ASSERT(grpc_tcp_server_add_port(s, (struct sockaddr *)&addr1, addr_len) ==
+             svr1_port);
+
+  /* Bad port_index. */
+  GPR_ASSERT(grpc_tcp_server_port_fd_count(s, 2) == 0);
+  GPR_ASSERT(grpc_tcp_server_port_fd(s, 2, 0) < 0);
+
+  /* Bad fd_index. */
+  GPR_ASSERT(grpc_tcp_server_port_fd(s, 0, 100) < 0);
+  GPR_ASSERT(grpc_tcp_server_port_fd(s, 1, 100) < 0);
+
+  /* Got at least one fd per port. */
+  svr_fd_count = grpc_tcp_server_port_fd_count(s, 0);
+  GPR_ASSERT(svr_fd_count >= 1);
+  svr1_fd_count = grpc_tcp_server_port_fd_count(s, 1);
+  GPR_ASSERT(svr1_fd_count >= 1);
 
-  svrfd = grpc_tcp_server_get_fd(s, 0);
-  GPR_ASSERT(svrfd >= 0);
-  GPR_ASSERT(getsockname(svrfd, (struct sockaddr *)&addr, &addr_len) == 0);
-  GPR_ASSERT(addr_len <= sizeof(addr));
+  for (i = 0; i < svr_fd_count; ++i) {
+    int fd = grpc_tcp_server_port_fd(s, 0, i);
+    GPR_ASSERT(fd >= 0);
+    if (i == 0) {
+      GPR_ASSERT(getsockname(fd, (struct sockaddr *)&addr, &addr_len) == 0);
+      GPR_ASSERT(addr_len <= sizeof(addr));
+    }
+  }
+  for (i = 0; i < svr1_fd_count; ++i) {
+    int fd = grpc_tcp_server_port_fd(s, 1, i);
+    GPR_ASSERT(fd >= 0);
+    if (i == 0) {
+      GPR_ASSERT(getsockname(fd, (struct sockaddr *)&addr1, &addr_len) == 0);
+      GPR_ASSERT(addr_len <= sizeof(addr1));
+    }
+  }
 
   pollsets[0] = &g_pollset;
   grpc_tcp_server_start(&exec_ctx, s, pollsets, 1, on_connect, NULL);
 
-  gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
-
   for (i = 0; i < n; i++) {
-    deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10);
-
-    nconnects_before = g_nconnects;
-    clifd = socket(addr.ss_family, SOCK_STREAM, 0);
-    GPR_ASSERT(clifd >= 0);
-    gpr_log(GPR_DEBUG, "start connect");
-    GPR_ASSERT(connect(clifd, (struct sockaddr *)&addr, addr_len) == 0);
-
-    gpr_log(GPR_DEBUG, "wait");
-    while (g_nconnects == nconnects_before &&
-           gpr_time_cmp(deadline, gpr_now(deadline.clock_type)) > 0) {
-      grpc_pollset_worker worker;
-      grpc_pollset_work(&exec_ctx, &g_pollset, &worker,
-                        gpr_now(GPR_CLOCK_MONOTONIC), deadline);
-      gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
-      grpc_exec_ctx_finish(&exec_ctx);
-      gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
+    on_connect_result result;
+    int svr_fd;
+    on_connect_result_init(&result);
+    tcp_connect(&exec_ctx, (struct sockaddr *)&addr, addr_len, &result);
+    GPR_ASSERT(result.server_fd >= 0);
+    svr_fd = result.server_fd;
+    GPR_ASSERT(grpc_tcp_server_port_fd(s, result.port_index, result.fd_index) ==
+               result.server_fd);
+    GPR_ASSERT(result.port_index == 0);
+    GPR_ASSERT(result.fd_index < svr_fd_count);
+    GPR_ASSERT(result.server == s);
+    if (weak_ref.server == NULL) {
+      server_weak_ref_set(&weak_ref, result.server);
     }
-    gpr_log(GPR_DEBUG, "wait done");
+    grpc_tcp_server_unref(&exec_ctx, result.server);
 
-    GPR_ASSERT(g_nconnects == nconnects_before + 1);
-    close(clifd);
+    on_connect_result_init(&result);
+    tcp_connect(&exec_ctx, (struct sockaddr *)&addr1, addr_len, &result);
+    GPR_ASSERT(result.server_fd >= 0);
+    GPR_ASSERT(result.server_fd != svr_fd);
+    GPR_ASSERT(grpc_tcp_server_port_fd(s, result.port_index, result.fd_index) ==
+               result.server_fd);
+    GPR_ASSERT(result.port_index == 1);
+    GPR_ASSERT(result.fd_index < svr_fd_count);
+    GPR_ASSERT(result.server == s);
+    grpc_tcp_server_unref(&exec_ctx, result.server);
   }
 
-  gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
+  /* Weak ref to server valid until final unref. */
+  GPR_ASSERT(weak_ref.server != NULL);
+  GPR_ASSERT(grpc_tcp_server_port_fd(s, 0, 0) >= 0);
+
+  grpc_tcp_server_unref(&exec_ctx, s);
+
+  /* Weak ref lost. */
+  GPR_ASSERT(weak_ref.server == NULL);
 
-  grpc_tcp_server_destroy(&exec_ctx, s, NULL);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, int success) {
+static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool success) {
   grpc_pollset_destroy(p);
 }
 
@@ -172,7 +311,7 @@ int main(int argc, char **argv) {
   grpc_closure destroyed;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_test_init(argc, argv);
-  grpc_iomgr_init();
+  grpc_init();
   grpc_pollset_init(&g_pollset);
 
   test_no_op();
@@ -185,6 +324,6 @@ int main(int argc, char **argv) {
   grpc_closure_init(&destroyed, destroy_pollset, &g_pollset);
   grpc_pollset_shutdown(&exec_ctx, &g_pollset, &destroyed);
   grpc_exec_ctx_finish(&exec_ctx);
-  grpc_iomgr_shutdown();
+  grpc_shutdown();
   return 0;
 }
diff --git a/test/core/iomgr/timer_list_test.c b/test/core/iomgr/timer_list_test.c
index 63014c3939..15de87c5a1 100644
--- a/test/core/iomgr/timer_list_test.c
+++ b/test/core/iomgr/timer_list_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -43,7 +43,7 @@
 
 static int cb_called[MAX_CB][2];
 
-static void cb(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void cb(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   cb_called[(intptr_t)arg][success]++;
 }
 
diff --git a/test/core/iomgr/workqueue_test.c b/test/core/iomgr/workqueue_test.c
index d1f9dabc57..500170b542 100644
--- a/test/core/iomgr/workqueue_test.c
+++ b/test/core/iomgr/workqueue_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -40,7 +40,7 @@
 
 static grpc_pollset g_pollset;
 
-static void must_succeed(grpc_exec_ctx *exec_ctx, void *p, int success) {
+static void must_succeed(grpc_exec_ctx *exec_ctx, void *p, bool success) {
   GPR_ASSERT(success == 1);
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   *(int *)p = 1;
@@ -90,7 +90,7 @@ static void test_flush(void) {
   grpc_pollset_worker worker;
   grpc_closure_init(&c, must_succeed, &done);
 
-  grpc_exec_ctx_enqueue(&exec_ctx, &c, 1);
+  grpc_exec_ctx_enqueue(&exec_ctx, &c, true, NULL);
   grpc_workqueue_flush(&exec_ctx, wq);
   grpc_workqueue_add_to_pollset(&exec_ctx, wq, &g_pollset);
 
@@ -106,7 +106,7 @@ static void test_flush(void) {
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, int success) {
+static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool success) {
   grpc_pollset_destroy(p);
 }
 
diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c
index fb62bf4134..55ac31e62c 100644
--- a/test/core/security/oauth2_utils.c
+++ b/test/core/security/oauth2_utils.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -73,7 +73,7 @@ static void on_oauth2_response(grpc_exec_ctx *exec_ctx, void *user_data,
   gpr_mu_unlock(GRPC_POLLSET_MU(&request->pollset));
 }
 
-static void do_nothing(grpc_exec_ctx *exec_ctx, void *unused, int success) {}
+static void do_nothing(grpc_exec_ctx *exec_ctx, void *unused, bool success) {}
 
 char *grpc_test_fetch_oauth2_token_with_credentials(
     grpc_call_credentials *creds) {
diff --git a/test/core/security/secure_endpoint_test.c b/test/core/security/secure_endpoint_test.c
index 240c4596b4..fb4bd30e2d 100644
--- a/test/core/security/secure_endpoint_test.c
+++ b/test/core/security/secure_endpoint_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -138,7 +138,7 @@ static grpc_endpoint_test_config configs[] = {
      secure_endpoint_create_fixture_tcp_socketpair_leftover, clean_up},
 };
 
-static void inc_call_ctr(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void inc_call_ctr(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   ++*(int *)arg;
 }
 
@@ -171,7 +171,7 @@ static void test_leftover(grpc_endpoint_test_config config, size_t slice_size) {
   clean_up();
 }
 
-static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, int success) {
+static void destroy_pollset(grpc_exec_ctx *exec_ctx, void *p, bool success) {
   grpc_pollset_destroy(p);
 }
 
diff --git a/test/core/security/security_connector_test.c b/test/core/security/security_connector_test.c
index 0dcffa40ce..ee5435f01d 100644
--- a/test/core/security/security_connector_test.c
+++ b/test/core/security/security_connector_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -36,6 +36,9 @@
 
 #include "src/core/security/security_connector.h"
 #include "src/core/security/security_context.h"
+#include "src/core/support/env.h"
+#include "src/core/support/file.h"
+#include "src/core/support/string.h"
 #include "src/core/tsi/ssl_transport_security.h"
 #include "src/core/tsi/transport_security.h"
 #include "test/core/util/test_config.h"
@@ -44,6 +47,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
 static int check_transport_security_type(const grpc_auth_context *ctx) {
@@ -297,7 +301,66 @@ static void test_cn_and_multiple_sans_and_others_ssl_peer_to_auth_context(
   GRPC_AUTH_CONTEXT_UNREF(ctx, "test");
 }
 
-/* TODO(jboeuf): Unit-test tsi_shallow_peer_from_auth_context. */
+static const char *roots_for_override_api = "roots for override api";
+
+static grpc_ssl_roots_override_result override_roots_success(
+    char **pem_root_certs) {
+  *pem_root_certs = gpr_strdup(roots_for_override_api);
+  return GRPC_SSL_ROOTS_OVERRIDE_OK;
+}
+
+static grpc_ssl_roots_override_result override_roots_permanent_failure(
+    char **pem_root_certs) {
+  return GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY;
+}
+
+static void test_default_ssl_roots(void) {
+  const char *roots_for_env_var = "roots for env var";
+
+  char *roots_env_var_file_path;
+  FILE *roots_env_var_file =
+      gpr_tmpfile("test_roots_for_env_var", &roots_env_var_file_path);
+  fwrite(roots_for_env_var, 1, strlen(roots_for_env_var), roots_env_var_file);
+  fclose(roots_env_var_file);
+
+  /* First let's get the root through the override: set the env to an invalid
+     value. */
+  gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, "");
+  grpc_set_ssl_roots_override_callback(override_roots_success);
+  gpr_slice roots = grpc_get_default_ssl_roots_for_testing();
+  char *roots_contents = gpr_dump_slice(roots, GPR_DUMP_ASCII);
+  gpr_slice_unref(roots);
+  GPR_ASSERT(strcmp(roots_contents, roots_for_override_api) == 0);
+  gpr_free(roots_contents);
+
+  /* Now let's set the env var: We should get the contents pointed value
+     instead. */
+  gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, roots_env_var_file_path);
+  roots = grpc_get_default_ssl_roots_for_testing();
+  roots_contents = gpr_dump_slice(roots, GPR_DUMP_ASCII);
+  gpr_slice_unref(roots);
+  GPR_ASSERT(strcmp(roots_contents, roots_for_env_var) == 0);
+  gpr_free(roots_contents);
+
+  /* Now reset the env var. We should fall back to the value overridden using
+     the api. */
+  gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, "");
+  roots = grpc_get_default_ssl_roots_for_testing();
+  roots_contents = gpr_dump_slice(roots, GPR_DUMP_ASCII);
+  gpr_slice_unref(roots);
+  GPR_ASSERT(strcmp(roots_contents, roots_for_override_api) == 0);
+  gpr_free(roots_contents);
+
+  /* Now setup a permanent failure for the overridden roots and we should get
+     an empty slice. */
+  grpc_set_ssl_roots_override_callback(override_roots_permanent_failure);
+  roots = grpc_get_default_ssl_roots_for_testing();
+  GPR_ASSERT(GPR_SLICE_IS_EMPTY(roots));
+
+  /* Cleanup. */
+  remove(roots_env_var_file_path);
+  gpr_free(roots_env_var_file_path);
+}
 
 int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
@@ -308,6 +371,7 @@ int main(int argc, char **argv) {
   test_cn_and_one_san_ssl_peer_to_auth_context();
   test_cn_and_multiple_sans_ssl_peer_to_auth_context();
   test_cn_and_multiple_sans_and_others_ssl_peer_to_auth_context();
+  test_default_ssl_roots();
 
   grpc_shutdown();
   return 0;
diff --git a/test/core/support/alloc_test.c b/test/core/support/alloc_test.c
index a7051a46a1..6bdba8c390 100644
--- a/test/core/support/alloc_test.c
+++ b/test/core/support/alloc_test.c
@@ -39,7 +39,9 @@ static void *fake_malloc(size_t size) { return (void *)size; }
 
 static void *fake_realloc(void *addr, size_t size) { return (void *)size; }
 
-static void fake_free(void *addr) { *((intptr_t *)addr) = 0xdeadd00d; }
+static void fake_free(void *addr) {
+  *((intptr_t *)addr) = (intptr_t)0xdeadd00d;
+}
 
 static void test_custom_allocs() {
   const gpr_allocation_functions default_fns = gpr_get_allocation_functions();
@@ -52,7 +54,7 @@ static void test_custom_allocs() {
   GPR_ASSERT((void *)(size_t)0xcafed00d == gpr_realloc(0, 0xcafed00d));
 
   gpr_free(&addr_to_free);
-  GPR_ASSERT(addr_to_free == 0xdeadd00d);
+  GPR_ASSERT(addr_to_free == (intptr_t)0xdeadd00d);
 
   /* Restore and check we don't get funky values and that we don't leak */
   gpr_set_allocation_functions(default_fns);
diff --git a/test/core/support/avl_test.c b/test/core/support/avl_test.c
index 6530fe4269..d8d8b36806 100644
--- a/test/core/support/avl_test.c
+++ b/test/core/support/avl_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -3611,32 +3611,33 @@ static void test_badcase3(void) {
   gpr_avl_unref(avl);
 }
 
-static void test_stress(void) {
+static void test_stress(int amount_of_stress) {
   int added[1024];
   int i, j;
   int deletions = 0;
   gpr_avl avl;
 
-  gpr_log(GPR_DEBUG, "test_stress");
+  unsigned seed = (unsigned)time(NULL);
+
+  gpr_log(GPR_DEBUG, "test_stress amount=%d seed=%u", amount_of_stress, seed);
 
   srand((unsigned)time(NULL));
   avl = gpr_avl_create(&int_int_vtable);
 
   memset(added, 0, sizeof(added));
 
-  for (i = 1; deletions < 1000; i++) {
+  for (i = 1; deletions < amount_of_stress; i++) {
     int idx = rand() % (int)GPR_ARRAY_SIZE(added);
     GPR_ASSERT(i);
     if (rand() < RAND_MAX / 2) {
       added[idx] = i;
-      fprintf(stderr, "avl = gpr_avl_add(avl, box(%d), box(%d)); /* d=%d */\n",
-              idx, i, deletions);
+      printf("avl = gpr_avl_add(avl, box(%d), box(%d)); /* d=%d */\n", idx, i,
+             deletions);
       avl = gpr_avl_add(avl, box(idx), box(i));
     } else {
       deletions += (added[idx] != 0);
       added[idx] = 0;
-      fprintf(stderr, "avl = remove_int(avl, %d); /* d=%d */\n", idx,
-              deletions);
+      printf("avl = remove_int(avl, %d); /* d=%d */\n", idx, deletions);
       avl = remove_int(avl, idx);
     }
     for (j = 0; j < (int)GPR_ARRAY_SIZE(added); j++) {
@@ -3665,7 +3666,7 @@ int main(int argc, char *argv[]) {
   test_badcase1();
   test_badcase2();
   test_badcase3();
-  test_stress();
+  test_stress(10);
 
   return 0;
 }
diff --git a/test/core/surface/lame_client_test.c b/test/core/surface/lame_client_test.c
index e03cce1322..79e53cb422 100644
--- a/test/core/surface/lame_client_test.c
+++ b/test/core/surface/lame_client_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -47,13 +47,13 @@ grpc_closure transport_op_cb;
 
 static void *tag(intptr_t x) { return (void *)x; }
 
-void verify_connectivity(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+void verify_connectivity(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   grpc_transport_op *op = arg;
   GPR_ASSERT(GRPC_CHANNEL_FATAL_FAILURE == *op->connectivity_state);
   GPR_ASSERT(success);
 }
 
-void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, int success) {}
+void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, bool success) {}
 
 void test_transport_op(grpc_channel *channel) {
   grpc_transport_op op;
diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c
index 1e1524d098..579faa4441 100644
--- a/test/core/surface/public_headers_must_be_c89.c
+++ b/test/core/surface/public_headers_must_be_c89.c
@@ -37,6 +37,21 @@
 #include <grpc/compression.h>
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
+#include <grpc/impl/codegen/alloc.h>
+#include <grpc/impl/codegen/atm.h>
+#include <grpc/impl/codegen/byte_buffer.h>
+#include <grpc/impl/codegen/compression_types.h>
+#include <grpc/impl/codegen/connectivity_state.h>
+#include <grpc/impl/codegen/grpc_types.h>
+#include <grpc/impl/codegen/log.h>
+#include <grpc/impl/codegen/port_platform.h>
+#include <grpc/impl/codegen/propagation_bits.h>
+#include <grpc/impl/codegen/slice.h>
+#include <grpc/impl/codegen/slice_buffer.h>
+#include <grpc/impl/codegen/status.h>
+#include <grpc/impl/codegen/sync.h>
+#include <grpc/impl/codegen/sync_generic.h>
+#include <grpc/impl/codegen/time.h>
 #include <grpc/status.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/atm.h>
diff --git a/test/core/transport/chttp2/hpack_table_test.c b/test/core/transport/chttp2/hpack_table_test.c
index 39f4174eec..3c5f2e4e31 100644
--- a/test/core/transport/chttp2/hpack_table_test.c
+++ b/test/core/transport/chttp2/hpack_table_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -236,7 +236,7 @@ static void test_find(void) {
 
   /* overflow the string buffer, check find still works */
   for (i = 0; i < 10000; i++) {
-    gpr_ltoa(i, buffer);
+    int64_ttoa(i, buffer);
     elem = grpc_mdelem_from_strings("test", buffer);
     GPR_ASSERT(grpc_chttp2_hptbl_add(&tbl, elem));
     GRPC_MDELEM_UNREF(elem);
@@ -256,7 +256,7 @@ static void test_find(void) {
 
   for (i = 0; i < tbl.num_ents; i++) {
     uint32_t expect = 9999 - i;
-    gpr_ltoa(expect, buffer);
+    int64_ttoa(expect, buffer);
 
     r = find_simple(&tbl, "test", buffer);
     GPR_ASSERT(r.index == i + 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY);
diff --git a/test/core/transport/chttp2/timeout_encoding_test.c b/test/core/transport/chttp2/timeout_encoding_test.c
index ba6c3191f1..f0e8ec386f 100644
--- a/test/core/transport/chttp2/timeout_encoding_test.c
+++ b/test/core/transport/chttp2/timeout_encoding_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -126,8 +126,13 @@ void test_decoding(void) {
   decode_suite('S', gpr_time_from_seconds);
   decode_suite('M', gpr_time_from_minutes);
   decode_suite('H', gpr_time_from_hours);
+  assert_decodes_as("1000000000S",
+                    gpr_time_from_seconds(1000 * 1000 * 1000, GPR_TIMESPAN));
   assert_decodes_as("1000000000000000000000u",
                     gpr_inf_future(GPR_CLOCK_REALTIME));
+  assert_decodes_as("1000000001S", gpr_inf_future(GPR_CLOCK_REALTIME));
+  assert_decodes_as("2000000001S", gpr_inf_future(GPR_CLOCK_REALTIME));
+  assert_decodes_as("9999999999S", gpr_inf_future(GPR_CLOCK_REALTIME));
 }
 
 void test_decoding_fails(void) {
diff --git a/test/core/transport/connectivity_state_test.c b/test/core/transport/connectivity_state_test.c
index 34ab45d260..4b2d0aa44a 100644
--- a/test/core/transport/connectivity_state_test.c
+++ b/test/core/transport/connectivity_state_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -43,13 +43,13 @@
 
 int g_counter;
 
-static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void must_succeed(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   GPR_ASSERT(success);
   GPR_ASSERT(arg == THE_ARG);
   g_counter++;
 }
 
-static void must_fail(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void must_fail(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   GPR_ASSERT(!success);
   GPR_ASSERT(arg == THE_ARG);
   g_counter++;
diff --git a/test/core/util/port_posix.c b/test/core/util/port_posix.c
index 732a51c5cb..4b31f810e5 100644
--- a/test/core/util/port_posix.c
+++ b/test/core/util/port_posix.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -37,11 +37,12 @@
 
 #include "test/core/util/port.h"
 
+#include <errno.h>
+#include <math.h>
 #include <netinet/in.h>
-#include <sys/socket.h>
 #include <stdio.h>
-#include <errno.h>
 #include <string.h>
+#include <sys/socket.h>
 #include <unistd.h>
 
 #include <grpc/grpc.h>
@@ -73,7 +74,7 @@ typedef struct freereq {
 } freereq;
 
 static void destroy_pollset_and_shutdown(grpc_exec_ctx *exec_ctx, void *p,
-                                         int success) {
+                                         bool success) {
   grpc_pollset_destroy(p);
   grpc_shutdown();
 }
@@ -229,10 +230,10 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg,
     grpc_httpcli_request req;
     memset(&req, 0, sizeof(req));
     GPR_ASSERT(pr->retries < 10);
+    sleep(1 + (unsigned)(pow(1.3, pr->retries) * rand() / RAND_MAX));
     pr->retries++;
     req.host = pr->server;
     req.path = "/get";
-    sleep(1);
     grpc_httpcli_get(exec_ctx, pr->ctx, &pr->pollset, &req,
                      GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10), got_port_from_server,
                      pr);
diff --git a/test/core/util/port_windows.c b/test/core/util/port_windows.c
index 60ad9b4f2a..85d7c0ce07 100644
--- a/test/core/util/port_windows.c
+++ b/test/core/util/port_windows.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -152,7 +152,7 @@ static void got_port_from_server(grpc_exec_ctx *exec_ctx, void *arg,
 }
 
 static void destroy_pollset_and_shutdown(grpc_exec_ctx *exec_ctx, void *p,
-                                         int success) {
+                                         bool success) {
   grpc_pollset_destroy(p);
   grpc_shutdown();
 }
diff --git a/test/core/util/reconnect_server.c b/test/core/util/reconnect_server.c
index 28e521221b..57225aa8a3 100644
--- a/test/core/util/reconnect_server.c
+++ b/test/core/util/reconnect_server.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -66,7 +66,8 @@ static void pretty_print_backoffs(reconnect_server *server) {
   }
 }
 
-static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp) {
+static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp,
+                       grpc_tcp_server_acceptor *acceptor) {
   char *peer;
   char *last_colon;
   reconnect_server *server = (reconnect_server *)arg;
diff --git a/test/core/util/test_tcp_server.c b/test/core/util/test_tcp_server.c
index 53b574d285..66470c0288 100644
--- a/test/core/util/test_tcp_server.c
+++ b/test/core/util/test_tcp_server.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -45,10 +45,17 @@
 #include "src/core/iomgr/tcp_server.h"
 #include "test/core/util/port.h"
 
+static void on_server_destroyed(grpc_exec_ctx *exec_ctx, void *data,
+                                bool success) {
+  test_tcp_server *server = data;
+  server->shutdown = 1;
+}
+
 void test_tcp_server_init(test_tcp_server *server,
                           grpc_tcp_server_cb on_connect, void *user_data) {
   grpc_init();
   server->tcp_server = NULL;
+  grpc_closure_init(&server->shutdown_complete, on_server_destroyed, server);
   server->shutdown = 0;
   grpc_pollset_init(&server->pollset);
   server->pollsets[0] = &server->pollset;
@@ -58,7 +65,6 @@ void test_tcp_server_init(test_tcp_server *server,
 
 void test_tcp_server_start(test_tcp_server *server, int port) {
   struct sockaddr_in addr;
-  grpc_tcp_listener *listener;
   int port_added;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
 
@@ -66,9 +72,9 @@ void test_tcp_server_start(test_tcp_server *server, int port) {
   addr.sin_port = htons((uint16_t)port);
   memset(&addr.sin_addr, 0, sizeof(addr.sin_addr));
 
-  server->tcp_server = grpc_tcp_server_create();
-  listener = grpc_tcp_server_add_port(server->tcp_server, &addr, sizeof(addr));
-  port_added = grpc_tcp_listener_get_port(listener);
+  server->tcp_server = grpc_tcp_server_create(&server->shutdown_complete);
+  port_added =
+      grpc_tcp_server_add_port(server->tcp_server, &addr, sizeof(addr));
   GPR_ASSERT(port_added == port);
 
   grpc_tcp_server_start(&exec_ctx, server->tcp_server, server->pollsets, 1,
@@ -91,22 +97,14 @@ void test_tcp_server_poll(test_tcp_server *server, int seconds) {
   grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void on_server_destroyed(grpc_exec_ctx *exec_ctx, void *data,
-                                int success) {
-  test_tcp_server *server = data;
-  server->shutdown = 1;
-}
-
-static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, int success) {}
+static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, bool success) {}
 
 void test_tcp_server_destroy(test_tcp_server *server) {
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   gpr_timespec shutdown_deadline;
-  grpc_closure server_shutdown_cb;
   grpc_closure do_nothing_cb;
-  grpc_closure_init(&server_shutdown_cb, on_server_destroyed, server);
+  grpc_tcp_server_unref(&exec_ctx, server->tcp_server);
   grpc_closure_init(&do_nothing_cb, do_nothing, NULL);
-  grpc_tcp_server_destroy(&exec_ctx, server->tcp_server, &server_shutdown_cb);
   shutdown_deadline = gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
                                    gpr_time_from_seconds(5, GPR_TIMESPAN));
   while (!server->shutdown &&
diff --git a/test/core/util/test_tcp_server.h b/test/core/util/test_tcp_server.h
index deb65eef11..51119cf6c8 100644
--- a/test/core/util/test_tcp_server.h
+++ b/test/core/util/test_tcp_server.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -39,6 +39,7 @@
 
 typedef struct test_tcp_server {
   grpc_tcp_server *tcp_server;
+  grpc_closure shutdown_complete;
   int shutdown;
   grpc_pollset pollset;
   grpc_pollset *pollsets[1];
diff --git a/test/cpp/common/alarm_test.cc b/test/cpp/common/alarm_test.cc
index d41a25a63c..09df6852a5 100644
--- a/test/cpp/common/alarm_test.cc
+++ b/test/cpp/common/alarm_test.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,6 +35,7 @@
 #include <grpc++/completion_queue.h>
 #include <gtest/gtest.h>
 
+#include <grpc++/completion_queue.h>
 #include "test/core/util/test_config.h"
 
 namespace grpc {
diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc
index cfda571326..0616cc07ee 100644
--- a/test/cpp/end2end/async_end2end_test.cc
+++ b/test/cpp/end2end/async_end2end_test.cc
@@ -180,21 +180,11 @@ class AsyncEnd2endTest : public ::testing::TestWithParam<bool> {
     int port = grpc_pick_unused_port_or_die();
     server_address_ << "localhost:" << port;
 
-    // It is currently unsupported to mix sync and async services
-    // in the same server, so first test that (for coverage)
-    ServerBuilder build_bad;
-    build_bad.AddListeningPort(server_address_.str(),
-                               grpc::InsecureServerCredentials());
-    build_bad.RegisterAsyncService(&service_);
-    grpc::testing::EchoTestService::Service sync_service;
-    build_bad.RegisterService(&sync_service);
-    GPR_ASSERT(build_bad.BuildAndStart() == nullptr);
-
     // Setup server
     ServerBuilder builder;
     builder.AddListeningPort(server_address_.str(),
                              grpc::InsecureServerCredentials());
-    builder.RegisterAsyncService(&service_);
+    builder.RegisterService(&service_);
     cq_ = builder.AddCompletionQueue();
     server_ = builder.BuildAndStart();
   }
diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc
index f8027bcf0b..3ad09aca4c 100644
--- a/test/cpp/end2end/end2end_test.cc
+++ b/test/cpp/end2end/end2end_test.cc
@@ -244,7 +244,8 @@ class TestServiceImpl : public ::grpc::testing::EchoTestService::Service {
           gpr_time_from_micros(request->param().server_cancel_after_us(),
                                GPR_TIMESPAN)));
       return Status::CANCELLED;
-    } else {
+    } else if (!request->has_param() ||
+               !request->param().skip_cancelled_check()) {
       EXPECT_FALSE(context->IsCancelled());
     }
 
@@ -451,13 +452,18 @@ class End2endTest : public ::testing::TestWithParam<TestScenario> {
   TestServiceImplDupPkg dup_pkg_service_;
 };
 
-static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs) {
+static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs,
+                    bool with_binary_metadata) {
   EchoRequest request;
   EchoResponse response;
   request.set_message("Hello hello hello hello");
 
   for (int i = 0; i < num_rpcs; ++i) {
     ClientContext context;
+    if (with_binary_metadata) {
+      char bytes[8] = {'\0', '\1', '\2', '\3', '\4', '\5', '\6', (char)i};
+      context.AddMetadata("custom-bin", grpc::string(bytes, 8));
+    }
     context.set_compression_algorithm(GRPC_COMPRESS_GZIP);
     Status s = stub->Echo(&context, request, &response);
     EXPECT_EQ(response.message(), request.message());
@@ -465,6 +471,30 @@ static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs) {
   }
 }
 
+TEST_P(End2endTest, MultipleRpcsWithVariedBinaryMetadataValue) {
+  ResetStub();
+  std::vector<std::thread*> threads;
+  for (int i = 0; i < 10; ++i) {
+    threads.push_back(new std::thread(SendRpc, stub_.get(), 10, true));
+  }
+  for (int i = 0; i < 10; ++i) {
+    threads[i]->join();
+    delete threads[i];
+  }
+}
+
+TEST_P(End2endTest, MultipleRpcs) {
+  ResetStub();
+  std::vector<std::thread*> threads;
+  for (int i = 0; i < 10; ++i) {
+    threads.push_back(new std::thread(SendRpc, stub_.get(), 10, false));
+  }
+  for (int i = 0; i < 10; ++i) {
+    threads[i]->join();
+    delete threads[i];
+  }
+}
+
 TEST_P(End2endTest, RequestStreamOneRequest) {
   ResetStub();
   EchoRequest request;
@@ -802,14 +832,14 @@ class ProxyEnd2endTest : public End2endTest {
 
 TEST_P(ProxyEnd2endTest, SimpleRpc) {
   ResetStub();
-  SendRpc(stub_.get(), 1);
+  SendRpc(stub_.get(), 1, false);
 }
 
 TEST_P(ProxyEnd2endTest, MultipleRpcs) {
   ResetStub();
   std::vector<std::thread*> threads;
   for (int i = 0; i < 10; ++i) {
-    threads.push_back(new std::thread(SendRpc, stub_.get(), 10));
+    threads.push_back(new std::thread(SendRpc, stub_.get(), 10, false));
   }
   for (int i = 0; i < 10; ++i) {
     threads[i]->join();
@@ -823,6 +853,7 @@ TEST_P(ProxyEnd2endTest, RpcDeadlineExpires) {
   EchoRequest request;
   EchoResponse response;
   request.set_message("Hello");
+  request.mutable_param()->set_skip_cancelled_check(true);
 
   ClientContext context;
   std::chrono::system_clock::time_point deadline =
diff --git a/test/cpp/end2end/generic_end2end_test.cc b/test/cpp/end2end/generic_end2end_test.cc
index c5d9de3f29..4e6d50ea80 100644
--- a/test/cpp/end2end/generic_end2end_test.cc
+++ b/test/cpp/end2end/generic_end2end_test.cc
@@ -51,6 +51,7 @@
 #include "src/proto/grpc/testing/echo.grpc.pb.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
+#include "test/cpp/util/byte_buffer_proto_helper.h"
 
 using grpc::testing::EchoRequest;
 using grpc::testing::EchoResponse;
@@ -70,29 +71,9 @@ void verify_ok(CompletionQueue* cq, int i, bool expect_ok) {
   EXPECT_EQ(tag(i), got_tag);
 }
 
-bool ParseFromByteBuffer(ByteBuffer* buffer, grpc::protobuf::Message* message) {
-  std::vector<Slice> slices;
-  buffer->Dump(&slices);
-  grpc::string buf;
-  buf.reserve(buffer->Length());
-  for (auto s = slices.begin(); s != slices.end(); s++) {
-    buf.append(reinterpret_cast<const char*>(s->begin()), s->size());
-  }
-  return message->ParseFromString(buf);
-}
-
-std::unique_ptr<ByteBuffer> SerializeToByteBuffer(
-    grpc::protobuf::Message* message) {
-  grpc::string buf;
-  message->SerializeToString(&buf);
-  gpr_slice s = gpr_slice_from_copied_string(buf.c_str());
-  Slice slice(s, Slice::STEAL_REF);
-  return std::unique_ptr<ByteBuffer>(new ByteBuffer(&slice, 1));
-}
-
 class GenericEnd2endTest : public ::testing::Test {
  protected:
-  GenericEnd2endTest() : generic_service_("*"), server_host_("localhost") {}
+  GenericEnd2endTest() : server_host_("localhost") {}
 
   void SetUp() GRPC_OVERRIDE {
     int port = grpc_pick_unused_port_or_die();
diff --git a/test/cpp/end2end/hybrid_end2end_test.cc b/test/cpp/end2end/hybrid_end2end_test.cc
new file mode 100644
index 0000000000..f8405627f9
--- /dev/null
+++ b/test/cpp/end2end/hybrid_end2end_test.cc
@@ -0,0 +1,556 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <memory>
+#include <thread>
+
+#include <grpc++/channel.h>
+#include <grpc++/client_context.h>
+#include <grpc++/create_channel.h>
+#include <grpc++/generic/async_generic_service.h>
+#include <grpc++/server.h>
+#include <grpc++/server_builder.h>
+#include <grpc++/server_context.h>
+#include <grpc/grpc.h>
+#include <gtest/gtest.h>
+
+#include "src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h"
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
+#include "test/core/util/port.h"
+#include "test/core/util/test_config.h"
+#include "test/cpp/end2end/test_service_impl.h"
+#include "test/cpp/util/byte_buffer_proto_helper.h"
+
+namespace grpc {
+namespace testing {
+
+namespace {
+
+void* tag(int i) { return (void*)(intptr_t)i; }
+
+bool VerifyReturnSuccess(CompletionQueue* cq, int i) {
+  void* got_tag;
+  bool ok;
+  EXPECT_TRUE(cq->Next(&got_tag, &ok));
+  EXPECT_EQ(tag(i), got_tag);
+  return ok;
+}
+
+void Verify(CompletionQueue* cq, int i, bool expect_ok) {
+  EXPECT_EQ(expect_ok, VerifyReturnSuccess(cq, i));
+}
+
+// Handlers to handle async request at a server. To be run in a separate thread.
+template <class Service>
+void HandleEcho(Service* service, ServerCompletionQueue* cq, bool dup_service) {
+  ServerContext srv_ctx;
+  grpc::ServerAsyncResponseWriter<EchoResponse> response_writer(&srv_ctx);
+  EchoRequest recv_request;
+  EchoResponse send_response;
+  service->RequestEcho(&srv_ctx, &recv_request, &response_writer, cq, cq,
+                       tag(1));
+  Verify(cq, 1, true);
+  send_response.set_message(recv_request.message());
+  if (dup_service) {
+    send_response.mutable_message()->append("_dup");
+  }
+  response_writer.Finish(send_response, Status::OK, tag(2));
+  Verify(cq, 2, true);
+}
+
+template <class Service>
+void HandleClientStreaming(Service* service, ServerCompletionQueue* cq) {
+  ServerContext srv_ctx;
+  EchoRequest recv_request;
+  EchoResponse send_response;
+  ServerAsyncReader<EchoResponse, EchoRequest> srv_stream(&srv_ctx);
+  service->RequestRequestStream(&srv_ctx, &srv_stream, cq, cq, tag(1));
+  Verify(cq, 1, true);
+  int i = 1;
+  do {
+    i++;
+    send_response.mutable_message()->append(recv_request.message());
+    srv_stream.Read(&recv_request, tag(i));
+  } while (VerifyReturnSuccess(cq, i));
+  srv_stream.Finish(send_response, Status::OK, tag(100));
+  Verify(cq, 100, true);
+}
+
+template <class Service>
+void HandleServerStreaming(Service* service, ServerCompletionQueue* cq) {
+  ServerContext srv_ctx;
+  EchoRequest recv_request;
+  EchoResponse send_response;
+  ServerAsyncWriter<EchoResponse> srv_stream(&srv_ctx);
+  service->RequestResponseStream(&srv_ctx, &recv_request, &srv_stream, cq, cq,
+                                 tag(1));
+  Verify(cq, 1, true);
+  send_response.set_message(recv_request.message() + "0");
+  srv_stream.Write(send_response, tag(2));
+  Verify(cq, 2, true);
+  send_response.set_message(recv_request.message() + "1");
+  srv_stream.Write(send_response, tag(3));
+  Verify(cq, 3, true);
+  send_response.set_message(recv_request.message() + "2");
+  srv_stream.Write(send_response, tag(4));
+  Verify(cq, 4, true);
+  srv_stream.Finish(Status::OK, tag(5));
+  Verify(cq, 5, true);
+}
+
+void HandleGenericEcho(GenericServerAsyncReaderWriter* stream,
+                       CompletionQueue* cq) {
+  ByteBuffer recv_buffer;
+  stream->Read(&recv_buffer, tag(2));
+  Verify(cq, 2, true);
+  EchoRequest recv_request;
+  EXPECT_TRUE(ParseFromByteBuffer(&recv_buffer, &recv_request));
+  EchoResponse send_response;
+  send_response.set_message(recv_request.message());
+  auto send_buffer = SerializeToByteBuffer(&send_response);
+  stream->Write(*send_buffer, tag(3));
+  Verify(cq, 3, true);
+  stream->Finish(Status::OK, tag(4));
+  Verify(cq, 4, true);
+}
+
+void HandleGenericRequestStream(GenericServerAsyncReaderWriter* stream,
+                                CompletionQueue* cq) {
+  ByteBuffer recv_buffer;
+  EchoRequest recv_request;
+  EchoResponse send_response;
+  int i = 1;
+  while (true) {
+    i++;
+    stream->Read(&recv_buffer, tag(i));
+    if (!VerifyReturnSuccess(cq, i)) {
+      break;
+    }
+    EXPECT_TRUE(ParseFromByteBuffer(&recv_buffer, &recv_request));
+    send_response.mutable_message()->append(recv_request.message());
+  }
+  auto send_buffer = SerializeToByteBuffer(&send_response);
+  stream->Write(*send_buffer, tag(99));
+  Verify(cq, 99, true);
+  stream->Finish(Status::OK, tag(100));
+  Verify(cq, 100, true);
+}
+
+// Request and handle one generic call.
+void HandleGenericCall(AsyncGenericService* service,
+                       ServerCompletionQueue* cq) {
+  GenericServerContext srv_ctx;
+  GenericServerAsyncReaderWriter stream(&srv_ctx);
+  service->RequestCall(&srv_ctx, &stream, cq, cq, tag(1));
+  Verify(cq, 1, true);
+  if (srv_ctx.method() == "/grpc.testing.EchoTestService/Echo") {
+    HandleGenericEcho(&stream, cq);
+  } else if (srv_ctx.method() ==
+             "/grpc.testing.EchoTestService/RequestStream") {
+    HandleGenericRequestStream(&stream, cq);
+  } else {  // other methods not handled yet.
+    gpr_log(GPR_ERROR, "method: %s", srv_ctx.method().c_str());
+    GPR_ASSERT(0);
+  }
+}
+
+class TestServiceImplDupPkg
+    : public ::grpc::testing::duplicate::EchoTestService::Service {
+ public:
+  Status Echo(ServerContext* context, const EchoRequest* request,
+              EchoResponse* response) GRPC_OVERRIDE {
+    response->set_message(request->message() + "_dup");
+    return Status::OK;
+  }
+};
+
+class HybridEnd2endTest : public ::testing::Test {
+ protected:
+  HybridEnd2endTest() {}
+
+  void SetUpServer(::grpc::Service* service1, ::grpc::Service* service2,
+                   AsyncGenericService* generic_service) {
+    int port = grpc_pick_unused_port_or_die();
+    server_address_ << "localhost:" << port;
+
+    // Setup server
+    ServerBuilder builder;
+    builder.AddListeningPort(server_address_.str(),
+                             grpc::InsecureServerCredentials());
+    builder.RegisterService(service1);
+    if (service2) {
+      builder.RegisterService(service2);
+    }
+    if (generic_service) {
+      builder.RegisterAsyncGenericService(generic_service);
+    }
+    // Create a separate cq for each potential handler.
+    for (int i = 0; i < 5; i++) {
+      cqs_.push_back(std::move(builder.AddCompletionQueue()));
+    }
+    server_ = builder.BuildAndStart();
+  }
+
+  void TearDown() GRPC_OVERRIDE {
+    if (server_) {
+      server_->Shutdown();
+    }
+    void* ignored_tag;
+    bool ignored_ok;
+    for (auto it = cqs_.begin(); it != cqs_.end(); ++it) {
+      (*it)->Shutdown();
+      while ((*it)->Next(&ignored_tag, &ignored_ok))
+        ;
+    }
+  }
+
+  void ResetStub() {
+    std::shared_ptr<Channel> channel =
+        CreateChannel(server_address_.str(), InsecureChannelCredentials());
+    stub_ = grpc::testing::EchoTestService::NewStub(channel);
+  }
+
+  // Test all rpc methods.
+  void TestAllMethods() {
+    SendEcho();
+    SendSimpleClientStreaming();
+    SendSimpleServerStreaming();
+    SendBidiStreaming();
+  }
+
+  void SendEcho() {
+    EchoRequest send_request;
+    EchoResponse recv_response;
+    ClientContext cli_ctx;
+    send_request.set_message("Hello");
+    Status recv_status = stub_->Echo(&cli_ctx, send_request, &recv_response);
+    EXPECT_EQ(send_request.message(), recv_response.message());
+    EXPECT_TRUE(recv_status.ok());
+  }
+
+  void SendEchoToDupService() {
+    std::shared_ptr<Channel> channel =
+        CreateChannel(server_address_.str(), InsecureChannelCredentials());
+    auto stub = grpc::testing::duplicate::EchoTestService::NewStub(channel);
+    EchoRequest send_request;
+    EchoResponse recv_response;
+    ClientContext cli_ctx;
+    send_request.set_message("Hello");
+    Status recv_status = stub->Echo(&cli_ctx, send_request, &recv_response);
+    EXPECT_EQ(send_request.message() + "_dup", recv_response.message());
+    EXPECT_TRUE(recv_status.ok());
+  }
+
+  void SendSimpleClientStreaming() {
+    EchoRequest send_request;
+    EchoResponse recv_response;
+    grpc::string expected_message;
+    ClientContext cli_ctx;
+    send_request.set_message("Hello");
+    auto stream = stub_->RequestStream(&cli_ctx, &recv_response);
+    for (int i = 0; i < 5; i++) {
+      EXPECT_TRUE(stream->Write(send_request));
+      expected_message.append(send_request.message());
+    }
+    stream->WritesDone();
+    Status recv_status = stream->Finish();
+    EXPECT_EQ(expected_message, recv_response.message());
+    EXPECT_TRUE(recv_status.ok());
+  }
+
+  void SendSimpleServerStreaming() {
+    EchoRequest request;
+    EchoResponse response;
+    ClientContext context;
+    request.set_message("hello");
+
+    auto stream = stub_->ResponseStream(&context, request);
+    EXPECT_TRUE(stream->Read(&response));
+    EXPECT_EQ(response.message(), request.message() + "0");
+    EXPECT_TRUE(stream->Read(&response));
+    EXPECT_EQ(response.message(), request.message() + "1");
+    EXPECT_TRUE(stream->Read(&response));
+    EXPECT_EQ(response.message(), request.message() + "2");
+    EXPECT_FALSE(stream->Read(&response));
+
+    Status s = stream->Finish();
+    EXPECT_TRUE(s.ok());
+  }
+
+  void SendBidiStreaming() {
+    EchoRequest request;
+    EchoResponse response;
+    ClientContext context;
+    grpc::string msg("hello");
+
+    auto stream = stub_->BidiStream(&context);
+
+    request.set_message(msg + "0");
+    EXPECT_TRUE(stream->Write(request));
+    EXPECT_TRUE(stream->Read(&response));
+    EXPECT_EQ(response.message(), request.message());
+
+    request.set_message(msg + "1");
+    EXPECT_TRUE(stream->Write(request));
+    EXPECT_TRUE(stream->Read(&response));
+    EXPECT_EQ(response.message(), request.message());
+
+    request.set_message(msg + "2");
+    EXPECT_TRUE(stream->Write(request));
+    EXPECT_TRUE(stream->Read(&response));
+    EXPECT_EQ(response.message(), request.message());
+
+    stream->WritesDone();
+    EXPECT_FALSE(stream->Read(&response));
+    EXPECT_FALSE(stream->Read(&response));
+
+    Status s = stream->Finish();
+    EXPECT_TRUE(s.ok());
+  }
+
+  std::vector<std::unique_ptr<ServerCompletionQueue> > cqs_;
+  std::unique_ptr<grpc::testing::EchoTestService::Stub> stub_;
+  std::unique_ptr<Server> server_;
+  std::ostringstream server_address_;
+};
+
+TEST_F(HybridEnd2endTest, AsyncEcho) {
+  EchoTestService::WithAsyncMethod_Echo<TestServiceImpl> service;
+  SetUpServer(&service, nullptr, nullptr);
+  ResetStub();
+  std::thread echo_handler_thread(
+      [this, &service] { HandleEcho(&service, cqs_[0].get(), false); });
+  TestAllMethods();
+  echo_handler_thread.join();
+}
+
+TEST_F(HybridEnd2endTest, AsyncEchoRequestStream) {
+  EchoTestService::WithAsyncMethod_RequestStream<
+      EchoTestService::WithAsyncMethod_Echo<TestServiceImpl> > service;
+  SetUpServer(&service, nullptr, nullptr);
+  ResetStub();
+  std::thread echo_handler_thread(
+      [this, &service] { HandleEcho(&service, cqs_[0].get(), false); });
+  std::thread request_stream_handler_thread(
+      [this, &service] { HandleClientStreaming(&service, cqs_[1].get()); });
+  TestAllMethods();
+  echo_handler_thread.join();
+  request_stream_handler_thread.join();
+}
+
+TEST_F(HybridEnd2endTest, AsyncRequestStreamResponseStream) {
+  EchoTestService::WithAsyncMethod_RequestStream<
+      EchoTestService::WithAsyncMethod_ResponseStream<TestServiceImpl> >
+      service;
+  SetUpServer(&service, nullptr, nullptr);
+  ResetStub();
+  std::thread response_stream_handler_thread(
+      [this, &service] { HandleServerStreaming(&service, cqs_[0].get()); });
+  std::thread request_stream_handler_thread(
+      [this, &service] { HandleClientStreaming(&service, cqs_[1].get()); });
+  TestAllMethods();
+  response_stream_handler_thread.join();
+  request_stream_handler_thread.join();
+}
+
+// Add a second service with one sync method.
+TEST_F(HybridEnd2endTest, AsyncRequestStreamResponseStream_SyncDupService) {
+  EchoTestService::WithAsyncMethod_RequestStream<
+      EchoTestService::WithAsyncMethod_ResponseStream<TestServiceImpl> >
+      service;
+  TestServiceImplDupPkg dup_service;
+  SetUpServer(&service, &dup_service, nullptr);
+  ResetStub();
+  std::thread response_stream_handler_thread(
+      [this, &service] { HandleServerStreaming(&service, cqs_[0].get()); });
+  std::thread request_stream_handler_thread(
+      [this, &service] { HandleClientStreaming(&service, cqs_[1].get()); });
+  TestAllMethods();
+  SendEchoToDupService();
+  response_stream_handler_thread.join();
+  request_stream_handler_thread.join();
+}
+
+// Add a second service with one async method.
+TEST_F(HybridEnd2endTest, AsyncRequestStreamResponseStream_AsyncDupService) {
+  EchoTestService::WithAsyncMethod_RequestStream<
+      EchoTestService::WithAsyncMethod_ResponseStream<TestServiceImpl> >
+      service;
+  duplicate::EchoTestService::AsyncService dup_service;
+  SetUpServer(&service, &dup_service, nullptr);
+  ResetStub();
+  std::thread response_stream_handler_thread(
+      [this, &service] { HandleServerStreaming(&service, cqs_[0].get()); });
+  std::thread request_stream_handler_thread(
+      [this, &service] { HandleClientStreaming(&service, cqs_[1].get()); });
+  std::thread echo_handler_thread(
+      [this, &dup_service] { HandleEcho(&dup_service, cqs_[2].get(), true); });
+  TestAllMethods();
+  SendEchoToDupService();
+  response_stream_handler_thread.join();
+  request_stream_handler_thread.join();
+  echo_handler_thread.join();
+}
+
+TEST_F(HybridEnd2endTest, GenericEcho) {
+  EchoTestService::WithGenericMethod_Echo<TestServiceImpl> service;
+  AsyncGenericService generic_service;
+  SetUpServer(&service, nullptr, &generic_service);
+  ResetStub();
+  std::thread generic_handler_thread([this, &generic_service] {
+    HandleGenericCall(&generic_service, cqs_[0].get());
+  });
+  TestAllMethods();
+  generic_handler_thread.join();
+}
+
+TEST_F(HybridEnd2endTest, GenericEchoAsyncRequestStream) {
+  EchoTestService::WithAsyncMethod_RequestStream<
+      EchoTestService::WithGenericMethod_Echo<TestServiceImpl> > service;
+  AsyncGenericService generic_service;
+  SetUpServer(&service, nullptr, &generic_service);
+  ResetStub();
+  std::thread generic_handler_thread([this, &generic_service] {
+    HandleGenericCall(&generic_service, cqs_[0].get());
+  });
+  std::thread request_stream_handler_thread(
+      [this, &service] { HandleClientStreaming(&service, cqs_[1].get()); });
+  TestAllMethods();
+  generic_handler_thread.join();
+  request_stream_handler_thread.join();
+}
+
+// Add a second service with one sync method.
+TEST_F(HybridEnd2endTest, GenericEchoAsyncRequestStream_SyncDupService) {
+  EchoTestService::WithAsyncMethod_RequestStream<
+      EchoTestService::WithGenericMethod_Echo<TestServiceImpl> > service;
+  AsyncGenericService generic_service;
+  TestServiceImplDupPkg dup_service;
+  SetUpServer(&service, &dup_service, &generic_service);
+  ResetStub();
+  std::thread generic_handler_thread([this, &generic_service] {
+    HandleGenericCall(&generic_service, cqs_[0].get());
+  });
+  std::thread request_stream_handler_thread(
+      [this, &service] { HandleClientStreaming(&service, cqs_[1].get()); });
+  TestAllMethods();
+  SendEchoToDupService();
+  generic_handler_thread.join();
+  request_stream_handler_thread.join();
+}
+
+// Add a second service with one async method.
+TEST_F(HybridEnd2endTest, GenericEchoAsyncRequestStream_AsyncDupService) {
+  EchoTestService::WithAsyncMethod_RequestStream<
+      EchoTestService::WithGenericMethod_Echo<TestServiceImpl> > service;
+  AsyncGenericService generic_service;
+  duplicate::EchoTestService::AsyncService dup_service;
+  SetUpServer(&service, &dup_service, &generic_service);
+  ResetStub();
+  std::thread generic_handler_thread([this, &generic_service] {
+    HandleGenericCall(&generic_service, cqs_[0].get());
+  });
+  std::thread request_stream_handler_thread(
+      [this, &service] { HandleClientStreaming(&service, cqs_[1].get()); });
+  std::thread echo_handler_thread(
+      [this, &dup_service] { HandleEcho(&dup_service, cqs_[2].get(), true); });
+  TestAllMethods();
+  SendEchoToDupService();
+  generic_handler_thread.join();
+  request_stream_handler_thread.join();
+  echo_handler_thread.join();
+}
+
+TEST_F(HybridEnd2endTest, GenericEchoAsyncRequestStreamResponseStream) {
+  EchoTestService::WithAsyncMethod_RequestStream<
+      EchoTestService::WithGenericMethod_Echo<
+          EchoTestService::WithAsyncMethod_ResponseStream<TestServiceImpl> > >
+      service;
+  AsyncGenericService generic_service;
+  SetUpServer(&service, nullptr, &generic_service);
+  ResetStub();
+  std::thread generic_handler_thread([this, &generic_service] {
+    HandleGenericCall(&generic_service, cqs_[0].get());
+  });
+  std::thread request_stream_handler_thread(
+      [this, &service] { HandleClientStreaming(&service, cqs_[1].get()); });
+  std::thread response_stream_handler_thread(
+      [this, &service] { HandleServerStreaming(&service, cqs_[2].get()); });
+  TestAllMethods();
+  generic_handler_thread.join();
+  request_stream_handler_thread.join();
+  response_stream_handler_thread.join();
+}
+
+TEST_F(HybridEnd2endTest, GenericEchoRequestStreamAsyncResponseStream) {
+  EchoTestService::WithGenericMethod_RequestStream<
+      EchoTestService::WithGenericMethod_Echo<
+          EchoTestService::WithAsyncMethod_ResponseStream<TestServiceImpl> > >
+      service;
+  AsyncGenericService generic_service;
+  SetUpServer(&service, nullptr, &generic_service);
+  ResetStub();
+  std::thread generic_handler_thread([this, &generic_service] {
+    HandleGenericCall(&generic_service, cqs_[0].get());
+  });
+  std::thread generic_handler_thread2([this, &generic_service] {
+    HandleGenericCall(&generic_service, cqs_[1].get());
+  });
+  std::thread response_stream_handler_thread(
+      [this, &service] { HandleServerStreaming(&service, cqs_[2].get()); });
+  TestAllMethods();
+  generic_handler_thread.join();
+  generic_handler_thread2.join();
+  response_stream_handler_thread.join();
+}
+
+// If WithGenericMethod is called and no generic service is registered, the
+// server will fail to build.
+TEST_F(HybridEnd2endTest, GenericMethodWithoutGenericService) {
+  EchoTestService::WithGenericMethod_RequestStream<
+      EchoTestService::WithGenericMethod_Echo<
+          EchoTestService::WithAsyncMethod_ResponseStream<TestServiceImpl> > >
+      service;
+  SetUpServer(&service, nullptr, nullptr);
+  EXPECT_EQ(nullptr, server_.get());
+}
+
+}  // namespace
+}  // namespace testing
+}  // namespace grpc
+
+int main(int argc, char** argv) {
+  grpc_test_init(argc, argv);
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/test/cpp/end2end/test_service_impl.cc b/test/cpp/end2end/test_service_impl.cc
new file mode 100644
index 0000000000..c9a32ecf5a
--- /dev/null
+++ b/test/cpp/end2end/test_service_impl.cc
@@ -0,0 +1,199 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "test/cpp/end2end/test_service_impl.h"
+
+#include <grpc++/security/credentials.h>
+#include <grpc++/server_context.h>
+#include <grpc/grpc.h>
+#include <gtest/gtest.h>
+
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
+#include "test/cpp/util/string_ref_helper.h"
+
+using std::chrono::system_clock;
+
+namespace grpc {
+namespace testing {
+namespace {
+
+// When echo_deadline is requested, deadline seen in the ServerContext is set in
+// the response in seconds.
+void MaybeEchoDeadline(ServerContext* context, const EchoRequest* request,
+                       EchoResponse* response) {
+  if (request->has_param() && request->param().echo_deadline()) {
+    gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_REALTIME);
+    if (context->deadline() != system_clock::time_point::max()) {
+      Timepoint2Timespec(context->deadline(), &deadline);
+    }
+    response->mutable_param()->set_request_deadline(deadline.tv_sec);
+  }
+}
+
+void CheckServerAuthContext(const ServerContext* context,
+                            const grpc::string& expected_client_identity) {
+  std::shared_ptr<const AuthContext> auth_ctx = context->auth_context();
+  std::vector<grpc::string_ref> ssl =
+      auth_ctx->FindPropertyValues("transport_security_type");
+  EXPECT_EQ(1u, ssl.size());
+  EXPECT_EQ("ssl", ToString(ssl[0]));
+  if (expected_client_identity.length() == 0) {
+    EXPECT_TRUE(auth_ctx->GetPeerIdentityPropertyName().empty());
+    EXPECT_TRUE(auth_ctx->GetPeerIdentity().empty());
+    EXPECT_FALSE(auth_ctx->IsPeerAuthenticated());
+  } else {
+    auto identity = auth_ctx->GetPeerIdentity();
+    EXPECT_TRUE(auth_ctx->IsPeerAuthenticated());
+    EXPECT_EQ(1u, identity.size());
+    EXPECT_EQ(expected_client_identity, identity[0]);
+  }
+}
+}  // namespace
+
+Status TestServiceImpl::Echo(ServerContext* context, const EchoRequest* request,
+                             EchoResponse* response) {
+  response->set_message(request->message());
+  MaybeEchoDeadline(context, request, response);
+  if (host_) {
+    response->mutable_param()->set_host(*host_);
+  }
+  if (request->has_param() && request->param().client_cancel_after_us()) {
+    {
+      std::unique_lock<std::mutex> lock(mu_);
+      signal_client_ = true;
+    }
+    while (!context->IsCancelled()) {
+      gpr_sleep_until(gpr_time_add(
+          gpr_now(GPR_CLOCK_REALTIME),
+          gpr_time_from_micros(request->param().client_cancel_after_us(),
+                               GPR_TIMESPAN)));
+    }
+    return Status::CANCELLED;
+  } else if (request->has_param() &&
+             request->param().server_cancel_after_us()) {
+    gpr_sleep_until(gpr_time_add(
+        gpr_now(GPR_CLOCK_REALTIME),
+        gpr_time_from_micros(request->param().server_cancel_after_us(),
+                             GPR_TIMESPAN)));
+    return Status::CANCELLED;
+  } else {
+    EXPECT_FALSE(context->IsCancelled());
+  }
+
+  if (request->has_param() && request->param().echo_metadata()) {
+    const std::multimap<grpc::string_ref, grpc::string_ref>& client_metadata =
+        context->client_metadata();
+    for (
+        std::multimap<grpc::string_ref, grpc::string_ref>::const_iterator iter =
+            client_metadata.begin();
+        iter != client_metadata.end(); ++iter) {
+      context->AddTrailingMetadata(ToString(iter->first),
+                                   ToString(iter->second));
+    }
+  }
+  if (request->has_param() &&
+      (request->param().expected_client_identity().length() > 0 ||
+       request->param().check_auth_context())) {
+    CheckServerAuthContext(context,
+                           request->param().expected_client_identity());
+  }
+  if (request->has_param() && request->param().response_message_length() > 0) {
+    response->set_message(
+        grpc::string(request->param().response_message_length(), '\0'));
+  }
+  if (request->has_param() && request->param().echo_peer()) {
+    response->mutable_param()->set_peer(context->peer());
+  }
+  return Status::OK;
+}
+
+// Unimplemented is left unimplemented to test the returned error.
+
+Status TestServiceImpl::RequestStream(ServerContext* context,
+                                      ServerReader<EchoRequest>* reader,
+                                      EchoResponse* response) {
+  EchoRequest request;
+  response->set_message("");
+  int cancel_after_reads = 0;
+  const std::multimap<grpc::string_ref, grpc::string_ref>&
+      client_initial_metadata = context->client_metadata();
+  if (client_initial_metadata.find(kServerCancelAfterReads) !=
+      client_initial_metadata.end()) {
+    std::istringstream iss(ToString(
+        client_initial_metadata.find(kServerCancelAfterReads)->second));
+    iss >> cancel_after_reads;
+    gpr_log(GPR_INFO, "cancel_after_reads %d", cancel_after_reads);
+  }
+  while (reader->Read(&request)) {
+    if (cancel_after_reads == 1) {
+      gpr_log(GPR_INFO, "return cancel status");
+      return Status::CANCELLED;
+    } else if (cancel_after_reads > 0) {
+      cancel_after_reads--;
+    }
+    response->mutable_message()->append(request.message());
+  }
+  return Status::OK;
+}
+
+// Return 3 messages.
+// TODO(yangg) make it generic by adding a parameter into EchoRequest
+Status TestServiceImpl::ResponseStream(ServerContext* context,
+                                       const EchoRequest* request,
+                                       ServerWriter<EchoResponse>* writer) {
+  EchoResponse response;
+  response.set_message(request->message() + "0");
+  writer->Write(response);
+  response.set_message(request->message() + "1");
+  writer->Write(response);
+  response.set_message(request->message() + "2");
+  writer->Write(response);
+
+  return Status::OK;
+}
+
+Status TestServiceImpl::BidiStream(
+    ServerContext* context,
+    ServerReaderWriter<EchoResponse, EchoRequest>* stream) {
+  EchoRequest request;
+  EchoResponse response;
+  while (stream->Read(&request)) {
+    gpr_log(GPR_INFO, "recv msg %s", request.message().c_str());
+    response.set_message(request.message());
+    stream->Write(response);
+  }
+  return Status::OK;
+}
+
+}  // namespace testing
+}  // namespace grpc
diff --git a/test/cpp/end2end/test_service_impl.h b/test/cpp/end2end/test_service_impl.h
new file mode 100644
index 0000000000..2c35b5614c
--- /dev/null
+++ b/test/cpp/end2end/test_service_impl.h
@@ -0,0 +1,85 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#ifndef GRPC_TEST_CPP_END2END_TEST_SERVICE_IMPL_H
+#define GRPC_TEST_CPP_END2END_TEST_SERVICE_IMPL_H
+
+#include <memory>
+#include <mutex>
+
+#include <grpc++/server_context.h>
+#include <grpc/grpc.h>
+
+#include "src/proto/grpc/testing/echo.grpc.pb.h"
+
+namespace grpc {
+namespace testing {
+
+const char* const kServerCancelAfterReads = "cancel_after_reads";
+
+class TestServiceImpl : public ::grpc::testing::EchoTestService::Service {
+ public:
+  TestServiceImpl() : signal_client_(false), host_() {}
+  explicit TestServiceImpl(const grpc::string& host)
+      : signal_client_(false), host_(new grpc::string(host)) {}
+
+  Status Echo(ServerContext* context, const EchoRequest* request,
+              EchoResponse* response) GRPC_OVERRIDE;
+
+  // Unimplemented is left unimplemented to test the returned error.
+
+  Status RequestStream(ServerContext* context,
+                       ServerReader<EchoRequest>* reader,
+                       EchoResponse* response) GRPC_OVERRIDE;
+
+  Status ResponseStream(ServerContext* context, const EchoRequest* request,
+                        ServerWriter<EchoResponse>* writer) GRPC_OVERRIDE;
+
+  Status BidiStream(ServerContext* context,
+                    ServerReaderWriter<EchoResponse, EchoRequest>* stream)
+      GRPC_OVERRIDE;
+
+  bool signal_client() {
+    std::unique_lock<std::mutex> lock(mu_);
+    return signal_client_;
+  }
+
+ private:
+  bool signal_client_;
+  std::mutex mu_;
+  std::unique_ptr<grpc::string> host_;
+};
+
+}  // namespace testing
+}  // namespace grpc
+
+#endif  // GRPC_TEST_CPP_END2END_TEST_SERVICE_IMPL_H
diff --git a/test/cpp/qps/async_streaming_ping_pong_test.cc b/test/cpp/qps/async_streaming_ping_pong_test.cc
index 0acdf3affb..97499329c1 100644
--- a/test/cpp/qps/async_streaming_ping_pong_test.cc
+++ b/test/cpp/qps/async_streaming_ping_pong_test.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -58,7 +58,6 @@ static void RunAsyncStreamingPingPong() {
 
   ServerConfig server_config;
   server_config.set_server_type(ASYNC_SERVER);
-  server_config.set_host("localhost");
   server_config.set_async_server_threads(1);
 
   const auto result =
diff --git a/test/cpp/qps/async_unary_ping_pong_test.cc b/test/cpp/qps/async_unary_ping_pong_test.cc
index d21e116171..d801bddf4a 100644
--- a/test/cpp/qps/async_unary_ping_pong_test.cc
+++ b/test/cpp/qps/async_unary_ping_pong_test.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -58,7 +58,6 @@ static void RunAsyncUnaryPingPong() {
 
   ServerConfig server_config;
   server_config.set_server_type(ASYNC_SERVER);
-  server_config.set_host("localhost");
   server_config.set_async_server_threads(1);
 
   const auto result =
diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc
index f270cd0987..4229e1956e 100644
--- a/test/cpp/qps/client_async.cc
+++ b/test/cpp/qps/client_async.cc
@@ -43,16 +43,19 @@
 #include <vector>
 
 #include <gflags/gflags.h>
+#include <grpc++/channel.h>
+#include <grpc++/client_context.h>
 #include <grpc++/client_context.h>
 #include <grpc++/generic/generic_stub.h>
 #include <grpc/grpc.h>
+#include <grpc/support/cpu.h>
 #include <grpc/support/histogram.h>
 #include <grpc/support/log.h>
 
+#include "src/proto/grpc/testing/services.grpc.pb.h"
 #include "test/cpp/qps/client.h"
 #include "test/cpp/qps/timer.h"
 #include "test/cpp/util/create_test_channel.h"
-#include "src/proto/grpc/testing/services.grpc.pb.h"
 
 namespace grpc {
 namespace testing {
@@ -164,14 +167,15 @@ class AsyncClient : public ClientImpl<StubType, RequestType> {
               std::function<std::unique_ptr<StubType>(std::shared_ptr<Channel>)>
                   create_stub)
       : ClientImpl<StubType, RequestType>(config, create_stub),
+        num_async_threads_(NumThreads(config)),
         channel_lock_(new std::mutex[config.client_channels()]),
         contexts_(config.client_channels()),
         max_outstanding_per_channel_(config.outstanding_rpcs_per_channel()),
         channel_count_(config.client_channels()),
-        pref_channel_inc_(config.async_client_threads()) {
-    SetupLoadTest(config, config.async_client_threads());
+        pref_channel_inc_(num_async_threads_) {
+    SetupLoadTest(config, num_async_threads_);
 
-    for (int i = 0; i < config.async_client_threads(); i++) {
+    for (int i = 0; i < num_async_threads_; i++) {
       cli_cqs_.emplace_back(new CompletionQueue);
       if (!closed_loop_) {
         rpc_deadlines_.emplace_back();
@@ -324,6 +328,9 @@ class AsyncClient : public ClientImpl<StubType, RequestType> {
     return true;
   }
 
+ protected:
+  int num_async_threads_;
+
  private:
   class boolean {  // exists only to avoid data-race on vector<bool>
    public:
@@ -338,6 +345,15 @@ class AsyncClient : public ClientImpl<StubType, RequestType> {
    private:
     bool val_;
   };
+  static int NumThreads(const ClientConfig& config) {
+    int num_threads = config.async_client_threads();
+    if (num_threads <= 0) {  // Use dynamic sizing
+      num_threads = gpr_cpu_num_cores();
+      gpr_log(GPR_INFO, "Sizing client server to %d threads", num_threads);
+    }
+    return num_threads;
+  }
+
   std::vector<std::unique_ptr<CompletionQueue>> cli_cqs_;
 
   std::vector<deadline_list> rpc_deadlines_;  // per thread deadlines
@@ -363,7 +379,7 @@ class AsyncUnaryClient GRPC_FINAL
  public:
   explicit AsyncUnaryClient(const ClientConfig& config)
       : AsyncClient(config, SetupCtx, BenchmarkStubCreator) {
-    StartThreads(config.async_client_threads());
+    StartThreads(num_async_threads_);
   }
   ~AsyncUnaryClient() GRPC_OVERRIDE { EndThreads(); }
 
@@ -461,7 +477,7 @@ class AsyncStreamingClient GRPC_FINAL
     // async streaming currently only supports closed loop
     GPR_ASSERT(closed_loop_);
 
-    StartThreads(config.async_client_threads());
+    StartThreads(num_async_threads_);
   }
 
   ~AsyncStreamingClient() GRPC_OVERRIDE { EndThreads(); }
@@ -566,7 +582,7 @@ class GenericAsyncStreamingClient GRPC_FINAL
     // async streaming currently only supports closed loop
     GPR_ASSERT(closed_loop_);
 
-    StartThreads(config.async_client_threads());
+    StartThreads(num_async_threads_);
   }
 
   ~GenericAsyncStreamingClient() GRPC_OVERRIDE { EndThreads(); }
diff --git a/test/cpp/qps/client_sync.cc b/test/cpp/qps/client_sync.cc
index 92fbf240ce..d93537b279 100644
--- a/test/cpp/qps/client_sync.cc
+++ b/test/cpp/qps/client_sync.cc
@@ -41,6 +41,7 @@
 #include <vector>
 
 #include <gflags/gflags.h>
+#include <grpc++/channel.h>
 #include <grpc++/client_context.h>
 #include <grpc++/server.h>
 #include <grpc++/server_builder.h>
diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc
index acb265b308..490156aec2 100644
--- a/test/cpp/qps/driver.cc
+++ b/test/cpp/qps/driver.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,24 +31,25 @@
  *
  */
 
+#include <deque>
 #include <list>
 #include <thread>
-#include <deque>
 #include <vector>
 
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/host_port.h>
+#include <grpc++/channel.h>
 #include <grpc++/client_context.h>
 #include <grpc++/create_channel.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/host_port.h>
+#include <grpc/support/log.h>
 
 #include "src/core/support/env.h"
+#include "src/proto/grpc/testing/services.grpc.pb.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
 #include "test/cpp/qps/driver.h"
 #include "test/cpp/qps/histogram.h"
 #include "test/cpp/qps/qps_worker.h"
-#include "src/proto/grpc/testing/services.grpc.pb.h"
 
 using std::list;
 using std::thread;
@@ -142,6 +143,12 @@ std::unique_ptr<ScenarioResult> RunScenario(
     }
   }
 
+  // if num_clients is set to <=0, do dynamic sizing: all workers
+  // except for servers are clients
+  if (num_clients <= 0) {
+    num_clients = workers.size() - num_servers;
+  }
+
   // TODO(ctiller): support running multiple configurations, and binpack
   // client/server pairs
   // to available workers
@@ -161,6 +168,8 @@ std::unique_ptr<ScenarioResult> RunScenario(
   // where class contained in std::vector must have a copy constructor
   auto* servers = new ServerData[num_servers];
   for (size_t i = 0; i < num_servers; i++) {
+    gpr_log(GPR_INFO, "Starting server on %s (worker #%d)", workers[i].c_str(),
+            i);
     servers[i].stub = WorkerService::NewStub(
         CreateChannel(workers[i], InsecureChannelCredentials()));
     ServerArgs args;
@@ -188,6 +197,8 @@ std::unique_ptr<ScenarioResult> RunScenario(
   // where class contained in std::vector must have a copy constructor
   auto* clients = new ClientData[num_clients];
   for (size_t i = 0; i < num_clients; i++) {
+    gpr_log(GPR_INFO, "Starting client on %s (worker #%d)",
+            workers[i + num_servers].c_str(), i + num_servers);
     clients[i].stub = WorkerService::NewStub(
         CreateChannel(workers[i + num_servers], InsecureChannelCredentials()));
     ClientArgs args;
diff --git a/test/cpp/qps/generic_async_streaming_ping_pong_test.cc b/test/cpp/qps/generic_async_streaming_ping_pong_test.cc
index 2b2e1c820f..d9166ae210 100644
--- a/test/cpp/qps/generic_async_streaming_ping_pong_test.cc
+++ b/test/cpp/qps/generic_async_streaming_ping_pong_test.cc
@@ -60,8 +60,7 @@ static void RunGenericAsyncStreamingPingPong() {
   bbuf->set_req_size(0);
 
   ServerConfig server_config;
-  server_config.set_server_type(ASYNC_SERVER);
-  server_config.set_host("localhost");
+  server_config.set_server_type(ASYNC_GENERIC_SERVER);
   server_config.set_async_server_threads(1);
 
   const auto result =
diff --git a/test/cpp/qps/qps-sweep.sh b/test/cpp/qps/qps-sweep.sh
index 36ea974812..333f4bd7d0 100755
--- a/test/cpp/qps/qps-sweep.sh
+++ b/test/cpp/qps/qps-sweep.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-# Copyright 2015, Google Inc.
+# Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -37,22 +37,93 @@ fi
 
 bins=`find . .. ../.. ../../.. -name bins | head -1`
 
-for secure in true false
-do
-  for channels in 1 2 4 8
-  do
-    for client in SYNC_CLIENT ASYNC_CLIENT
-    do
-      for server in SYNC_SERVER ASYNC_SERVER
-      do
-        for rpc in UNARY STREAMING
-        do
-          echo "Test $rpc $client $server, $channels channels, secure=$secure"
-          "$bins"/opt/qps_driver --rpc_type=$rpc \
-              --client_type=$client --server_type=$server \
-	      --secure_test=$secure
-	done
-      done
-    done
-  done
+set -x
+
+big=65536
+half=`echo $QPS_WORKERS | awk -F, '{print int(NF/2)}'`
+
+for secure in true false; do
+  # Scenario 1: generic async streaming ping-pong (contentionless latency)
+  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=1 \
+    --client_channels=1 --bbuf_req_size=0 --bbuf_resp_size=0 \
+    --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
+    --num_servers=1 --num_clients=1
+
+  # Scenario 2: generic async streaming "unconstrained" (QPS)
+  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=100 \
+    --client_channels=64 --bbuf_req_size=0 --bbuf_resp_size=0 \
+    --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \
+    --num_servers=1 --num_clients=0
+
+  # Scenario 3: Latency at near-peak load (TBD)
+
+  # Scenario 4: Single-channel bidirectional throughput test (like TCP_STREAM).
+  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=100 \
+    --client_channels=1 --bbuf_req_size=$big --bbuf_resp_size=$big \
+    --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
+    --num_servers=1 --num_clients=1
+
+  # Scenario 5: Sync unary ping-pong with protobufs
+  "$bins"/opt/qps_driver --rpc_type=UNARY --client_type=SYNC_CLIENT \
+    --server_type=SYNC_SERVER --outstanding_rpcs_per_channel=1 \
+    --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \
+    --secure_test=$secure --num_servers=1 --num_clients=1
+
+  # Scenario 6: Sync streaming ping-pong with protobufs
+  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=SYNC_CLIENT \
+    --server_type=SYNC_SERVER --outstanding_rpcs_per_channel=1 \
+    --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \
+    --secure_test=$secure --num_servers=1 --num_clients=1
+
+  # Scenario 7: Async unary ping-pong with protobufs
+  "$bins"/opt/qps_driver --rpc_type=UNARY --client_type=ASYNC_CLIENT \
+    --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=1 \
+    --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \
+    --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
+    --num_servers=1 --num_clients=1
+
+  # Scenario 8: Async streaming ping-pong with protobufs
+  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+    --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=1 \
+    --client_channels=1 --simple_req_size=0 --simple_resp_size=0 \
+    --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
+    --num_servers=1 --num_clients=1
+
+  # Scenario 9: Crossbar QPS test
+  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=100 \
+    --client_channels=64 --bbuf_req_size=0 --bbuf_resp_size=0 \
+    --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \
+    --num_servers=$half --num_clients=0
+
+  # Scenario 10: Multi-channel bidir throughput test
+  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=100 \
+    --client_channels=64 --bbuf_req_size=$big --bbuf_resp_size=$big \
+    --async_client_threads=0 --async_server_threads=0 --secure_test=$secure \
+    --num_servers=1 --num_clients=1
+
+  # Scenario 11: Single-channel request throughput test
+  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=100 \
+    --client_channels=1 --bbuf_req_size=$big --bbuf_resp_size=0 \
+    --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
+    --num_servers=1 --num_clients=1
+
+  # Scenario 12: Single-channel response throughput test
+  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+    --server_type=ASYNC_GENERIC_SERVER --outstanding_rpcs_per_channel=100 \
+    --client_channels=1 --bbuf_req_size=0 --bbuf_resp_size=$big \
+    --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
+    --num_servers=1 --num_clients=1
+
+  # Scenario 13: Single-channel bidirectional protobuf throughput test
+  "$bins"/opt/qps_driver --rpc_type=STREAMING --client_type=ASYNC_CLIENT \
+    --server_type=ASYNC_SERVER --outstanding_rpcs_per_channel=100 \
+    --client_channels=1 --simple_req_size=$big --simple_resp_size=$big \
+    --async_client_threads=1 --async_server_threads=1 --secure_test=$secure \
+    --num_servers=1 --num_clients=1
 done
diff --git a/test/cpp/qps/qps_driver.cc b/test/cpp/qps/qps_driver.cc
index e770f92d3f..6462050b6c 100644
--- a/test/cpp/qps/qps_driver.cc
+++ b/test/cpp/qps/qps_driver.cc
@@ -179,7 +179,6 @@ static void QpsDriver() {
 
   ServerConfig server_config;
   server_config.set_server_type(server_type);
-  server_config.set_host("localhost");
   server_config.set_async_server_threads(FLAGS_async_server_threads);
 
   if (FLAGS_server_core_list.size() > 0) {
@@ -198,6 +197,13 @@ static void QpsDriver() {
     server_config.mutable_security_params()->CopyFrom(security);
   }
 
+  // Make sure that if we are performing a generic (bytebuf) test
+  // that we are also using async streaming
+  GPR_ASSERT(!client_config.payload_config().has_bytebuf_params() ||
+             (client_config.client_type() == ASYNC_CLIENT &&
+              client_config.rpc_type() == STREAMING &&
+              server_config.server_type() == ASYNC_GENERIC_SERVER));
+
   const auto result = RunScenario(
       client_config, FLAGS_num_clients, server_config, FLAGS_num_servers,
       FLAGS_warmup_seconds, FLAGS_benchmark_seconds, FLAGS_local_workers);
diff --git a/test/cpp/qps/qps_openloop_test.cc b/test/cpp/qps/qps_openloop_test.cc
index 51df79ef2f..fe5f685b6e 100644
--- a/test/cpp/qps/qps_openloop_test.cc
+++ b/test/cpp/qps/qps_openloop_test.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -59,7 +59,6 @@ static void RunQPS() {
 
   ServerConfig server_config;
   server_config.set_server_type(ASYNC_SERVER);
-  server_config.set_host("localhost");
   server_config.set_async_server_threads(4);
 
   const auto result =
diff --git a/test/cpp/qps/qps_test.cc b/test/cpp/qps/qps_test.cc
index 1f87d18137..15054db892 100644
--- a/test/cpp/qps/qps_test.cc
+++ b/test/cpp/qps/qps_test.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -58,7 +58,6 @@ static void RunQPS() {
 
   ServerConfig server_config;
   server_config.set_server_type(ASYNC_SERVER);
-  server_config.set_host("localhost");
   server_config.set_async_server_threads(8);
 
   const auto result =
diff --git a/test/cpp/qps/qps_test_with_poll.cc b/test/cpp/qps/qps_test_with_poll.cc
index dc800092db..8340a6386a 100644
--- a/test/cpp/qps/qps_test_with_poll.cc
+++ b/test/cpp/qps/qps_test_with_poll.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -62,7 +62,6 @@ static void RunQPS() {
 
   ServerConfig server_config;
   server_config.set_server_type(ASYNC_SERVER);
-  server_config.set_host("localhost");
   server_config.set_async_server_threads(4);
 
   const auto result =
diff --git a/test/cpp/qps/qps_worker.cc b/test/cpp/qps/qps_worker.cc
index d259c20fb7..5cb5850fd4 100644
--- a/test/cpp/qps/qps_worker.cc
+++ b/test/cpp/qps/qps_worker.cc
@@ -61,6 +61,11 @@ namespace grpc {
 namespace testing {
 
 static std::unique_ptr<Client> CreateClient(const ClientConfig& config) {
+  gpr_log(GPR_INFO, "Starting client of type %s %s %d",
+          ClientType_Name(config.client_type()).c_str(),
+          RpcType_Name(config.rpc_type()).c_str(),
+          config.payload_config().has_bytebuf_params());
+
   switch (config.client_type()) {
     case ClientType::SYNC_CLIENT:
       return (config.rpc_type() == RpcType::UNARY)
@@ -79,11 +84,20 @@ static std::unique_ptr<Client> CreateClient(const ClientConfig& config) {
 }
 
 static std::unique_ptr<Server> CreateServer(const ServerConfig& config) {
+  gpr_log(GPR_INFO, "Starting server of type %s",
+          ServerType_Name(config.server_type()).c_str());
+
+  if (config.core_limit() > 0) {
+    LimitCores(config.core_limit());
+  }
+
   switch (config.server_type()) {
     case ServerType::SYNC_SERVER:
       return CreateSynchronousServer(config);
     case ServerType::ASYNC_SERVER:
       return CreateAsyncServer(config);
+    case ServerType::ASYNC_GENERIC_SERVER:
+      return CreateAsyncGenericServer(config);
     default:
       abort();
   }
@@ -92,7 +106,8 @@ static std::unique_ptr<Server> CreateServer(const ServerConfig& config) {
 
 class WorkerServiceImpl GRPC_FINAL : public WorkerService::Service {
  public:
-  explicit WorkerServiceImpl() : acquired_(false) {}
+  explicit WorkerServiceImpl(int server_port)
+      : acquired_(false), server_port_(server_port) {}
 
   Status RunClient(ServerContext* ctx,
                    ServerReaderWriter<ClientStatus, ClientArgs>* stream)
@@ -163,22 +178,29 @@ class WorkerServiceImpl GRPC_FINAL : public WorkerService::Service {
     if (!args.has_setup()) {
       return Status(StatusCode::INVALID_ARGUMENT, "");
     }
+    gpr_log(GPR_INFO, "RunClientBody: about to create client");
     auto client = CreateClient(args.setup());
     if (!client) {
       return Status(StatusCode::INVALID_ARGUMENT, "");
     }
+    gpr_log(GPR_INFO, "RunClientBody: client created");
     ClientStatus status;
     if (!stream->Write(status)) {
       return Status(StatusCode::UNKNOWN, "");
     }
+    gpr_log(GPR_INFO, "RunClientBody: creation status reported");
     while (stream->Read(&args)) {
+      gpr_log(GPR_INFO, "RunClientBody: Message read");
       if (!args.has_mark()) {
+        gpr_log(GPR_INFO, "RunClientBody: Message is not a mark!");
         return Status(StatusCode::INVALID_ARGUMENT, "");
       }
       *status.mutable_stats() = client->Mark(args.mark().reset());
       stream->Write(status);
+      gpr_log(GPR_INFO, "RunClientBody: Mark response given");
     }
 
+    gpr_log(GPR_INFO, "RunClientBody: Returning");
     return Status::OK;
   }
 
@@ -191,33 +213,44 @@ class WorkerServiceImpl GRPC_FINAL : public WorkerService::Service {
     if (!args.has_setup()) {
       return Status(StatusCode::INVALID_ARGUMENT, "");
     }
+    if (server_port_ != 0) {
+      args.mutable_setup()->set_port(server_port_);
+    }
+    gpr_log(GPR_INFO, "RunServerBody: about to create server");
     auto server = CreateServer(args.setup());
     if (!server) {
       return Status(StatusCode::INVALID_ARGUMENT, "");
     }
+    gpr_log(GPR_INFO, "RunServerBody: server created");
     ServerStatus status;
     status.set_port(server->port());
     status.set_cores(server->cores());
     if (!stream->Write(status)) {
       return Status(StatusCode::UNKNOWN, "");
     }
+    gpr_log(GPR_INFO, "RunServerBody: creation status reported");
     while (stream->Read(&args)) {
+      gpr_log(GPR_INFO, "RunServerBody: Message read");
       if (!args.has_mark()) {
+        gpr_log(GPR_INFO, "RunServerBody: Message not a mark!");
         return Status(StatusCode::INVALID_ARGUMENT, "");
       }
       *status.mutable_stats() = server->Mark(args.mark().reset());
       stream->Write(status);
+      gpr_log(GPR_INFO, "RunServerBody: Mark response given");
     }
 
+    gpr_log(GPR_INFO, "RunServerBody: Returning");
     return Status::OK;
   }
 
   std::mutex mu_;
   bool acquired_;
+  int server_port_;
 };
 
-QpsWorker::QpsWorker(int driver_port) {
-  impl_.reset(new WorkerServiceImpl());
+QpsWorker::QpsWorker(int driver_port, int server_port) {
+  impl_.reset(new WorkerServiceImpl(server_port));
 
   char* server_address = NULL;
   gpr_join_host_port(&server_address, "::", driver_port);
diff --git a/test/cpp/qps/qps_worker.h b/test/cpp/qps/qps_worker.h
index 0db88ad3d1..27de69fa65 100644
--- a/test/cpp/qps/qps_worker.h
+++ b/test/cpp/qps/qps_worker.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -46,7 +46,7 @@ class WorkerServiceImpl;
 
 class QpsWorker {
  public:
-  explicit QpsWorker(int driver_port);
+  explicit QpsWorker(int driver_port, int server_port = 0);
   ~QpsWorker();
 
  private:
diff --git a/test/cpp/qps/secure_sync_unary_ping_pong_test.cc b/test/cpp/qps/secure_sync_unary_ping_pong_test.cc
index ce9f02cceb..359310b856 100644
--- a/test/cpp/qps/secure_sync_unary_ping_pong_test.cc
+++ b/test/cpp/qps/secure_sync_unary_ping_pong_test.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -57,7 +57,6 @@ static void RunSynchronousUnaryPingPong() {
 
   ServerConfig server_config;
   server_config.set_server_type(SYNC_SERVER);
-  server_config.set_host("localhost");
 
   // Set up security params
   SecurityParams security;
diff --git a/test/cpp/qps/server.h b/test/cpp/qps/server.h
index 6195e6ce63..bc6f9f99e3 100644
--- a/test/cpp/qps/server.h
+++ b/test/cpp/qps/server.h
@@ -121,6 +121,7 @@ class Server {
 
 std::unique_ptr<Server> CreateSynchronousServer(const ServerConfig& config);
 std::unique_ptr<Server> CreateAsyncServer(const ServerConfig& config);
+std::unique_ptr<Server> CreateAsyncGenericServer(const ServerConfig& config);
 
 }  // namespace testing
 }  // namespace grpc
diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc
index d530dac86b..1302d718f0 100644
--- a/test/cpp/qps/server_async.cc
+++ b/test/cpp/qps/server_async.cc
@@ -50,8 +50,8 @@
 #include <grpc/support/log.h>
 #include <gtest/gtest.h>
 
-#include "test/cpp/qps/server.h"
 #include "src/proto/grpc/testing/services.grpc.pb.h"
+#include "test/cpp/qps/server.h"
 
 namespace grpc {
 namespace testing {
@@ -76,7 +76,7 @@ class AsyncQpsServerTest : public Server {
       : Server(config) {
     char *server_address = NULL;
 
-    gpr_join_host_port(&server_address, config.host().c_str(), port());
+    gpr_join_host_port(&server_address, "::", port());
 
     ServerBuilder builder;
     builder.AddListeningPort(server_address,
@@ -85,7 +85,13 @@ class AsyncQpsServerTest : public Server {
 
     register_service(&builder, &async_service_);
 
-    for (int i = 0; i < config.async_server_threads(); i++) {
+    int num_threads = config.async_server_threads();
+    if (num_threads <= 0) {  // dynamic sizing
+      num_threads = cores();
+      gpr_log(GPR_INFO, "Sizing async server to %d threads", num_threads);
+    }
+
+    for (int i = 0; i < num_threads; i++) {
       srv_cqs_.emplace_back(builder.AddCompletionQueue());
     }
 
@@ -96,8 +102,8 @@ class AsyncQpsServerTest : public Server {
     auto process_rpc_bound =
         std::bind(process_rpc, config.payload_config(), _1, _2);
 
-    for (int i = 0; i < 10000 / config.async_server_threads(); i++) {
-      for (int j = 0; j < config.async_server_threads(); j++) {
+    for (int i = 0; i < 10000 / num_threads; i++) {
+      for (int j = 0; j < num_threads; j++) {
         if (request_unary_function) {
           auto request_unary =
               std::bind(request_unary_function, &async_service_, _1, _2, _3,
@@ -115,10 +121,10 @@ class AsyncQpsServerTest : public Server {
       }
     }
 
-    for (int i = 0; i < config.async_server_threads(); i++) {
+    for (int i = 0; i < num_threads; i++) {
       shutdown_state_.emplace_back(new PerThreadShutdownState());
     }
-    for (int i = 0; i < config.async_server_threads(); i++) {
+    for (int i = 0; i < num_threads; i++) {
       threads_.emplace_back(&AsyncQpsServerTest::ThreadFunc, this, i);
     }
   }
@@ -350,7 +356,7 @@ class AsyncQpsServerTest : public Server {
 
 static void RegisterBenchmarkService(ServerBuilder *builder,
                                      BenchmarkService::AsyncService *service) {
-  builder->RegisterAsyncService(service);
+  builder->RegisterService(service);
 }
 static void RegisterGenericService(ServerBuilder *builder,
                                    grpc::AsyncGenericService *service) {
@@ -373,7 +379,7 @@ static Status ProcessGenericRPC(const PayloadConfig &payload_config,
                                 const ByteBuffer *request,
                                 ByteBuffer *response) {
   int resp_size = payload_config.bytebuf_params().resp_size();
-  std::unique_ptr<char> buf(new char[resp_size]);
+  std::unique_ptr<char[]> buf(new char[resp_size]);
   gpr_slice s = gpr_slice_from_copied_buffer(buf.get(), resp_size);
   Slice slice(s, Slice::STEAL_REF);
   *response = ByteBuffer(&slice, 1);
diff --git a/test/cpp/qps/server_sync.cc b/test/cpp/qps/server_sync.cc
index 97a1ff5255..4b778820d0 100644
--- a/test/cpp/qps/server_sync.cc
+++ b/test/cpp/qps/server_sync.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -89,7 +89,7 @@ class SynchronousServer GRPC_FINAL : public grpc::testing::Server {
 
     char* server_address = NULL;
 
-    gpr_join_host_port(&server_address, config.host().c_str(), port());
+    gpr_join_host_port(&server_address, "::", port());
     builder.AddListeningPort(server_address,
                              Server::CreateServerCredentials(config));
     gpr_free(server_address);
diff --git a/test/cpp/qps/sync_streaming_ping_pong_test.cc b/test/cpp/qps/sync_streaming_ping_pong_test.cc
index dd8c682815..e02c14c926 100644
--- a/test/cpp/qps/sync_streaming_ping_pong_test.cc
+++ b/test/cpp/qps/sync_streaming_ping_pong_test.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -57,7 +57,6 @@ static void RunSynchronousStreamingPingPong() {
 
   ServerConfig server_config;
   server_config.set_server_type(SYNC_SERVER);
-  server_config.set_host("localhost");
 
   const auto result =
       RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
diff --git a/test/cpp/qps/sync_unary_ping_pong_test.cc b/test/cpp/qps/sync_unary_ping_pong_test.cc
index 2edb33ef01..9d363c04fb 100644
--- a/test/cpp/qps/sync_unary_ping_pong_test.cc
+++ b/test/cpp/qps/sync_unary_ping_pong_test.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -57,7 +57,6 @@ static void RunSynchronousUnaryPingPong() {
 
   ServerConfig server_config;
   server_config.set_server_type(SYNC_SERVER);
-  server_config.set_host("localhost");
 
   const auto result =
       RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
diff --git a/test/cpp/qps/worker.cc b/test/cpp/qps/worker.cc
index 430ffb7cdc..a1e73e9abe 100644
--- a/test/cpp/qps/worker.cc
+++ b/test/cpp/qps/worker.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -44,6 +44,7 @@
 #include "test/cpp/util/test_config.h"
 
 DEFINE_int32(driver_port, 0, "Port for communication with driver");
+DEFINE_int32(server_port, 0, "Port for operation as a server");
 
 static bool got_sigint = false;
 
@@ -53,7 +54,7 @@ namespace grpc {
 namespace testing {
 
 static void RunServer() {
-  QpsWorker worker(FLAGS_driver_port);
+  QpsWorker worker(FLAGS_driver_port, FLAGS_server_port);
 
   while (!got_sigint) {
     gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
diff --git a/test/cpp/util/byte_buffer_proto_helper.cc b/test/cpp/util/byte_buffer_proto_helper.cc
new file mode 100644
index 0000000000..2512c9bdf8
--- /dev/null
+++ b/test/cpp/util/byte_buffer_proto_helper.cc
@@ -0,0 +1,60 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "test/cpp/util/byte_buffer_proto_helper.h"
+
+namespace grpc {
+namespace testing {
+
+bool ParseFromByteBuffer(ByteBuffer* buffer, grpc::protobuf::Message* message) {
+  std::vector<Slice> slices;
+  buffer->Dump(&slices);
+  grpc::string buf;
+  buf.reserve(buffer->Length());
+  for (auto s = slices.begin(); s != slices.end(); s++) {
+    buf.append(reinterpret_cast<const char*>(s->begin()), s->size());
+  }
+  return message->ParseFromString(buf);
+}
+
+std::unique_ptr<ByteBuffer> SerializeToByteBuffer(
+    grpc::protobuf::Message* message) {
+  grpc::string buf;
+  message->SerializeToString(&buf);
+  gpr_slice s = gpr_slice_from_copied_string(buf.c_str());
+  Slice slice(s, Slice::STEAL_REF);
+  return std::unique_ptr<ByteBuffer>(new ByteBuffer(&slice, 1));
+}
+
+}  // namespace testing
+}  // namespace grpc
diff --git a/test/cpp/util/byte_buffer_proto_helper.h b/test/cpp/util/byte_buffer_proto_helper.h
new file mode 100644
index 0000000000..42cea59e33
--- /dev/null
+++ b/test/cpp/util/byte_buffer_proto_helper.h
@@ -0,0 +1,53 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_TEST_CPP_UTIL_BYTE_BUFFER_PROTO_HELPER_H
+#define GRPC_TEST_CPP_UTIL_BYTE_BUFFER_PROTO_HELPER_H
+
+#include <memory>
+
+#include <grpc++/support/byte_buffer.h>
+#include <grpc++/support/config_protobuf.h>
+
+namespace grpc {
+namespace testing {
+
+bool ParseFromByteBuffer(ByteBuffer* buffer, grpc::protobuf::Message* message);
+
+std::unique_ptr<ByteBuffer> SerializeToByteBuffer(
+    grpc::protobuf::Message* message);
+
+}  // namespace testing
+}  // namespace grpc
+
+#endif  // GRPC_TEST_CPP_UTIL_BYTE_BUFFER_PROTO_HELPER_H
diff --git a/test/cpp/util/cli_call.cc b/test/cpp/util/cli_call.cc
index 5e29e7a1d3..d452be360d 100644
--- a/test/cpp/util/cli_call.cc
+++ b/test/cpp/util/cli_call.cc
@@ -37,6 +37,7 @@
 
 #include <grpc++/channel.h>
 #include <grpc++/client_context.h>
+#include <grpc++/completion_queue.h>
 #include <grpc++/generic/generic_stub.h>
 #include <grpc++/support/byte_buffer.h>
 #include <grpc/grpc.h>
diff --git a/test/cpp/util/metrics_server.cc b/test/cpp/util/metrics_server.cc
index 40028d3957..07978d0bdb 100644
--- a/test/cpp/util/metrics_server.cc
+++ b/test/cpp/util/metrics_server.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -33,6 +33,7 @@
 
 #include "test/cpp/util/metrics_server.h"
 
+#include <grpc++/server.h>
 #include <grpc++/server_builder.h>
 
 #include "src/proto/grpc/testing/metrics.grpc.pb.h"