Merge github.com:grpc/grpc into kinetic-lopsided-poetry

30 files changed, 745 insertions, 181 deletions

diff --git a/test/core/end2end/dualstack_socket_test.c b/test/core/end2end/dualstack_socket_test.c
index 77bea2abab..c98baeffda 100644
--- a/test/core/end2end/dualstack_socket_test.c
+++ b/test/core/end2end/dualstack_socket_test.c
@@ -96,8 +96,8 @@ void test_connect(const char *server_host, const char *client_host, int port,
   cq = grpc_completion_queue_create();
   server = grpc_server_create(NULL);
   grpc_server_register_completion_queue(server, cq);
-  GPR_ASSERT((got_port = grpc_server_add_http2_port(server, server_hostport)) >
-             0);
+  GPR_ASSERT((got_port = grpc_server_add_insecure_http2_port(
+                  server, server_hostport)) > 0);
   if (port == 0) {
     port = got_port;
   } else {
diff --git a/test/core/end2end/fixtures/chttp2_fullstack.c b/test/core/end2end/fixtures/chttp2_fullstack.c
index 6647b949ba..53a6f0d7a5 100644
--- a/test/core/end2end/fixtures/chttp2_fullstack.c
+++ b/test/core/end2end/fixtures/chttp2_fullstack.c
@@ -84,7 +84,7 @@ void chttp2_init_server_fullstack(grpc_end2end_test_fixture *f,
   }
   f->server = grpc_server_create(server_args);
   grpc_server_register_completion_queue(f->server, f->cq);
-  GPR_ASSERT(grpc_server_add_http2_port(f->server, ffd->localaddr));
+  GPR_ASSERT(grpc_server_add_insecure_http2_port(f->server, ffd->localaddr));
   grpc_server_start(f->server);
 }
 
diff --git a/test/core/end2end/fixtures/chttp2_fullstack_compression.c b/test/core/end2end/fixtures/chttp2_fullstack_compression.c
index f3d1fa22dc..0ee24c01b5 100644
--- a/test/core/end2end/fixtures/chttp2_fullstack_compression.c
+++ b/test/core/end2end/fixtures/chttp2_fullstack_compression.c
@@ -99,7 +99,7 @@ void chttp2_init_server_fullstack_compression(grpc_end2end_test_fixture *f,
   }
   f->server = grpc_server_create(ffd->server_args_compression);
   grpc_server_register_completion_queue(f->server, f->cq);
-  GPR_ASSERT(grpc_server_add_http2_port(f->server, ffd->localaddr));
+  GPR_ASSERT(grpc_server_add_insecure_http2_port(f->server, ffd->localaddr));
   grpc_server_start(f->server);
 }
 
diff --git a/test/core/end2end/fixtures/chttp2_fullstack_uds_posix.c b/test/core/end2end/fixtures/chttp2_fullstack_uds_posix.c
index 89ad7b8c2d..20afdb868e 100644
--- a/test/core/end2end/fixtures/chttp2_fullstack_uds_posix.c
+++ b/test/core/end2end/fixtures/chttp2_fullstack_uds_posix.c
@@ -89,7 +89,7 @@ void chttp2_init_server_fullstack(grpc_end2end_test_fixture *f,
   }
   f->server = grpc_server_create(server_args);
   grpc_server_register_completion_queue(f->server, f->cq);
-  GPR_ASSERT(grpc_server_add_http2_port(f->server, ffd->localaddr));
+  GPR_ASSERT(grpc_server_add_insecure_http2_port(f->server, ffd->localaddr));
   grpc_server_start(f->server);
 }
 
diff --git a/test/core/end2end/fixtures/chttp2_fullstack_uds_posix_with_poll.c b/test/core/end2end/fixtures/chttp2_fullstack_uds_posix_with_poll.c
index a2ab25d886..8491ea6970 100644
--- a/test/core/end2end/fixtures/chttp2_fullstack_uds_posix_with_poll.c
+++ b/test/core/end2end/fixtures/chttp2_fullstack_uds_posix_with_poll.c
@@ -89,7 +89,7 @@ void chttp2_init_server_fullstack(grpc_end2end_test_fixture *f,
   }
   f->server = grpc_server_create(server_args);
   grpc_server_register_completion_queue(f->server, f->cq);
-  GPR_ASSERT(grpc_server_add_http2_port(f->server, ffd->localaddr));
+  GPR_ASSERT(grpc_server_add_insecure_http2_port(f->server, ffd->localaddr));
   grpc_server_start(f->server);
 }
 
diff --git a/test/core/end2end/fixtures/chttp2_fullstack_with_poll.c b/test/core/end2end/fixtures/chttp2_fullstack_with_poll.c
index 93786d0943..2a4835add1 100644
--- a/test/core/end2end/fixtures/chttp2_fullstack_with_poll.c
+++ b/test/core/end2end/fixtures/chttp2_fullstack_with_poll.c
@@ -83,7 +83,7 @@ void chttp2_init_server_fullstack(grpc_end2end_test_fixture *f,
   }
   f->server = grpc_server_create(server_args);
   grpc_server_register_completion_queue(f->server, f->cq);
-  GPR_ASSERT(grpc_server_add_http2_port(f->server, ffd->localaddr));
+  GPR_ASSERT(grpc_server_add_insecure_http2_port(f->server, ffd->localaddr));
   grpc_server_start(f->server);
 }
 
diff --git a/test/core/end2end/multiple_server_queues_test.c b/test/core/end2end/multiple_server_queues_test.c
index 208d42e6e7..7772d14ba5 100644
--- a/test/core/end2end/multiple_server_queues_test.c
+++ b/test/core/end2end/multiple_server_queues_test.c
@@ -45,7 +45,7 @@ int main(int argc, char **argv) {
   cq2 = grpc_completion_queue_create();
   server = grpc_server_create(NULL);
   grpc_server_register_completion_queue(server, cq1);
-  grpc_server_add_http2_port(server, "[::]:0");
+  grpc_server_add_insecure_http2_port(server, "[::]:0");
   grpc_server_register_completion_queue(server, cq2);
   grpc_server_start(server);
   grpc_server_shutdown_and_notify(server, cq2, NULL);
diff --git a/test/core/fling/server.c b/test/core/fling/server.c
index 8f349044d9..f445c68178 100644
--- a/test/core/fling/server.c
+++ b/test/core/fling/server.c
@@ -216,7 +216,7 @@ int main(int argc, char **argv) {
     grpc_server_credentials_release(ssl_creds);
   } else {
     server = grpc_server_create(NULL);
-    GPR_ASSERT(grpc_server_add_http2_port(server, addr));
+    GPR_ASSERT(grpc_server_add_insecure_http2_port(server, addr));
   }
   grpc_server_register_completion_queue(server, cq);
   grpc_server_start(server);
diff --git a/test/core/httpcli/httpcli_test.c b/test/core/httpcli/httpcli_test.c
index 034501ee69..8dddfbee98 100644
--- a/test/core/httpcli/httpcli_test.c
+++ b/test/core/httpcli/httpcli_test.c
@@ -64,7 +64,7 @@ static void on_finish(void *arg, const grpc_httpcli_response *response) {
   GPR_ASSERT(0 == memcmp(expect, response->body, response->body_length));
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   g_done = 1;
-  grpc_pollset_kick(&g_pollset);
+  grpc_pollset_kick(&g_pollset, NULL);
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
 }
 
@@ -87,7 +87,8 @@ static void test_get(int use_ssl, int port) {
                    (void *)42);
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   while (!g_done) {
-    grpc_pollset_work(&g_pollset, n_seconds_time(20));
+    grpc_pollset_worker worker;
+    grpc_pollset_work(&g_pollset, &worker, n_seconds_time(20));
   }
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
   gpr_free(host);
@@ -112,7 +113,8 @@ static void test_post(int use_ssl, int port) {
                     n_seconds_time(15), on_finish, (void *)42);
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   while (!g_done) {
-    grpc_pollset_work(&g_pollset, n_seconds_time(20));
+    grpc_pollset_worker worker;
+    grpc_pollset_work(&g_pollset, &worker, n_seconds_time(20));
   }
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
   gpr_free(host);
diff --git a/test/core/iomgr/endpoint_tests.c b/test/core/iomgr/endpoint_tests.c
index cb6adc58cf..8186c96da1 100644
--- a/test/core/iomgr/endpoint_tests.c
+++ b/test/core/iomgr/endpoint_tests.c
@@ -132,7 +132,7 @@ static void read_and_write_test_read_handler(void *data, gpr_slice *slices,
     gpr_log(GPR_INFO, "Read handler shutdown");
     gpr_mu_lock(GRPC_POLLSET_MU(g_pollset));
     state->read_done = 1;
-    grpc_pollset_kick(g_pollset);
+    grpc_pollset_kick(g_pollset, NULL);
     gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset));
     return;
   }
@@ -143,7 +143,7 @@ static void read_and_write_test_read_handler(void *data, gpr_slice *slices,
     gpr_log(GPR_INFO, "Read handler done");
     gpr_mu_lock(GRPC_POLLSET_MU(g_pollset));
     state->read_done = 1;
-    grpc_pollset_kick(g_pollset);
+    grpc_pollset_kick(g_pollset, NULL);
     gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset));
   } else {
     grpc_endpoint_notify_on_read(state->read_ep,
@@ -167,7 +167,7 @@ static void read_and_write_test_write_handler(void *data,
     gpr_log(GPR_INFO, "Write handler shutdown");
     gpr_mu_lock(GRPC_POLLSET_MU(g_pollset));
     state->write_done = 1;
-    grpc_pollset_kick(g_pollset);
+    grpc_pollset_kick(g_pollset, NULL);
     gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset));
     return;
   }
@@ -201,7 +201,7 @@ static void read_and_write_test_write_handler(void *data,
   gpr_log(GPR_INFO, "Write handler done");
   gpr_mu_lock(GRPC_POLLSET_MU(g_pollset));
   state->write_done = 1;
-  grpc_pollset_kick(g_pollset);
+  grpc_pollset_kick(g_pollset, NULL);
   gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset));
 }
 
@@ -254,8 +254,9 @@ static void read_and_write_test(grpc_endpoint_test_config config,
 
   gpr_mu_lock(GRPC_POLLSET_MU(g_pollset));
   while (!state.read_done || !state.write_done) {
+    grpc_pollset_worker worker;
     GPR_ASSERT(gpr_time_cmp(gpr_now(GPR_CLOCK_MONOTONIC), deadline) < 0);
-    grpc_pollset_work(g_pollset, deadline);
+    grpc_pollset_work(g_pollset, &worker, deadline);
   }
   gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset));
 
@@ -287,7 +288,7 @@ static void shutdown_during_write_test_read_handler(
     grpc_endpoint_destroy(st->ep);
     gpr_mu_lock(GRPC_POLLSET_MU(g_pollset));
     st->done = error;
-    grpc_pollset_kick(g_pollset);
+    grpc_pollset_kick(g_pollset, NULL);
     gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset));
   } else {
     grpc_endpoint_notify_on_read(
@@ -309,7 +310,7 @@ static void shutdown_during_write_test_write_handler(
   }
   gpr_mu_lock(GRPC_POLLSET_MU(g_pollset));
   st->done = 1;
-  grpc_pollset_kick(g_pollset);
+  grpc_pollset_kick(g_pollset, NULL);
   gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset));
 }
 
@@ -350,15 +351,17 @@ static void shutdown_during_write_test(grpc_endpoint_test_config config,
         deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(10);
         gpr_mu_lock(GRPC_POLLSET_MU(g_pollset));
         while (!write_st.done) {
+          grpc_pollset_worker worker;
           GPR_ASSERT(gpr_time_cmp(gpr_now(deadline.clock_type), deadline) < 0);
-          grpc_pollset_work(g_pollset, deadline);
+          grpc_pollset_work(g_pollset, &worker, deadline);
         }
         gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset));
         grpc_endpoint_destroy(write_st.ep);
         gpr_mu_lock(GRPC_POLLSET_MU(g_pollset));
         while (!read_st.done) {
+          grpc_pollset_worker worker;
           GPR_ASSERT(gpr_time_cmp(gpr_now(deadline.clock_type), deadline) < 0);
-          grpc_pollset_work(g_pollset, deadline);
+          grpc_pollset_work(g_pollset, &worker, deadline);
         }
         gpr_mu_unlock(GRPC_POLLSET_MU(g_pollset));
         gpr_free(slices);
diff --git a/test/core/iomgr/fd_posix_test.c b/test/core/iomgr/fd_posix_test.c
index 7d00c098cc..adcbcafdbb 100644
--- a/test/core/iomgr/fd_posix_test.c
+++ b/test/core/iomgr/fd_posix_test.c
@@ -179,7 +179,7 @@ static void listen_shutdown_cb(void *arg /*server*/, int success) {
 
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   sv->done = 1;
-  grpc_pollset_kick(&g_pollset);
+  grpc_pollset_kick(&g_pollset, NULL);
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
 }
 
@@ -249,7 +249,8 @@ static int server_start(server *sv) {
 static void server_wait_and_shutdown(server *sv) {
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   while (!sv->done) {
-    grpc_pollset_work(&g_pollset, gpr_inf_future(GPR_CLOCK_MONOTONIC));
+    grpc_pollset_worker worker;
+    grpc_pollset_work(&g_pollset, &worker, gpr_inf_future(GPR_CLOCK_MONOTONIC));
   }
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
 }
@@ -286,7 +287,7 @@ static void client_session_shutdown_cb(void *arg /*client*/, int success) {
   client *cl = arg;
   grpc_fd_orphan(cl->em_fd, NULL, "c");
   cl->done = 1;
-  grpc_pollset_kick(&g_pollset);
+  grpc_pollset_kick(&g_pollset, NULL);
 }
 
 /* Write as much as possible, then register notify_on_write. */
@@ -356,7 +357,8 @@ static void client_start(client *cl, int port) {
 static void client_wait_and_shutdown(client *cl) {
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   while (!cl->done) {
-    grpc_pollset_work(&g_pollset, gpr_inf_future(GPR_CLOCK_MONOTONIC));
+    grpc_pollset_worker worker;
+    grpc_pollset_work(&g_pollset, &worker, gpr_inf_future(GPR_CLOCK_MONOTONIC));
   }
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
 }
@@ -392,7 +394,7 @@ static void first_read_callback(void *arg /* fd_change_data */, int success) {
 
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   fdc->cb_that_ran = first_read_callback;
-  grpc_pollset_kick(&g_pollset);
+  grpc_pollset_kick(&g_pollset, NULL);
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
 }
 
@@ -401,7 +403,7 @@ static void second_read_callback(void *arg /* fd_change_data */, int success) {
 
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   fdc->cb_that_ran = second_read_callback;
-  grpc_pollset_kick(&g_pollset);
+  grpc_pollset_kick(&g_pollset, NULL);
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
 }
 
@@ -445,7 +447,8 @@ static void test_grpc_fd_change(void) {
   /* And now wait for it to run. */
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   while (a.cb_that_ran == NULL) {
-    grpc_pollset_work(&g_pollset, gpr_inf_future(GPR_CLOCK_MONOTONIC));
+    grpc_pollset_worker worker;
+    grpc_pollset_work(&g_pollset, &worker, gpr_inf_future(GPR_CLOCK_MONOTONIC));
   }
   GPR_ASSERT(a.cb_that_ran == first_read_callback);
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
@@ -463,7 +466,8 @@ static void test_grpc_fd_change(void) {
 
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   while (b.cb_that_ran == NULL) {
-    grpc_pollset_work(&g_pollset, gpr_inf_future(GPR_CLOCK_MONOTONIC));
+    grpc_pollset_worker worker;
+    grpc_pollset_work(&g_pollset, &worker, gpr_inf_future(GPR_CLOCK_MONOTONIC));
   }
   /* Except now we verify that second_read_callback ran instead */
   GPR_ASSERT(b.cb_that_ran == second_read_callback);
diff --git a/test/core/iomgr/poll_kick_posix_test.c b/test/core/iomgr/poll_kick_posix_test.c
deleted file mode 100644
index 3aa6807806..0000000000
--- a/test/core/iomgr/poll_kick_posix_test.c
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "src/core/iomgr/pollset_kick_posix.h"
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include "test/core/util/test_config.h"
-
-static void test_allocation(void) {
-  grpc_pollset_kick_state state;
-  grpc_pollset_kick_init(&state);
-  grpc_pollset_kick_destroy(&state);
-}
-
-static void test_non_kick(void) {
-  grpc_pollset_kick_state state;
-  grpc_kick_fd_info *kfd;
-
-  grpc_pollset_kick_init(&state);
-  kfd = grpc_pollset_kick_pre_poll(&state);
-  GPR_ASSERT(kfd != NULL);
-
-  grpc_pollset_kick_post_poll(&state, kfd);
-  grpc_pollset_kick_destroy(&state);
-}
-
-static void test_basic_kick(void) {
-  /* Kicked during poll */
-  grpc_pollset_kick_state state;
-  grpc_kick_fd_info *kfd;
-  grpc_pollset_kick_init(&state);
-
-  kfd = grpc_pollset_kick_pre_poll(&state);
-  GPR_ASSERT(kfd != NULL);
-
-  grpc_pollset_kick_kick(&state);
-
-  /* Now hypothetically we polled and found that we were kicked */
-  grpc_pollset_kick_consume(&state, kfd);
-
-  grpc_pollset_kick_post_poll(&state, kfd);
-
-  grpc_pollset_kick_destroy(&state);
-}
-
-static void test_non_poll_kick(void) {
-  /* Kick before entering poll */
-  grpc_pollset_kick_state state;
-  grpc_kick_fd_info *kfd;
-
-  grpc_pollset_kick_init(&state);
-
-  grpc_pollset_kick_kick(&state);
-  kfd = grpc_pollset_kick_pre_poll(&state);
-  GPR_ASSERT(kfd == NULL);
-  grpc_pollset_kick_destroy(&state);
-}
-
-#define GRPC_MAX_CACHED_PIPES 50
-
-static void test_over_free(void) {
-  /* Check high watermark pipe free logic */
-  int i;
-  grpc_kick_fd_info **kfds =
-      gpr_malloc(sizeof(grpc_kick_fd_info *) * GRPC_MAX_CACHED_PIPES);
-  grpc_pollset_kick_state state;
-  grpc_pollset_kick_init(&state);
-  for (i = 0; i < GRPC_MAX_CACHED_PIPES; ++i) {
-    kfds[i] = grpc_pollset_kick_pre_poll(&state);
-    GPR_ASSERT(kfds[i] != NULL);
-  }
-
-  for (i = 0; i < GRPC_MAX_CACHED_PIPES; ++i) {
-    grpc_pollset_kick_post_poll(&state, kfds[i]);
-  }
-  grpc_pollset_kick_destroy(&state);
-  gpr_free(kfds);
-}
-
-static void run_tests(void) {
-  test_allocation();
-  test_basic_kick();
-  test_non_poll_kick();
-  test_non_kick();
-  test_over_free();
-}
-
-int main(int argc, char **argv) {
-  grpc_test_init(argc, argv);
-
-  grpc_pollset_kick_global_init();
-  run_tests();
-  grpc_pollset_kick_global_destroy();
-
-  grpc_pollset_kick_global_init_fallback_fd();
-  run_tests();
-  grpc_pollset_kick_global_destroy();
-  return 0;
-}
diff --git a/test/core/iomgr/tcp_client_posix_test.c b/test/core/iomgr/tcp_client_posix_test.c
index 38b7b5909d..07bbe1f402 100644
--- a/test/core/iomgr/tcp_client_posix_test.c
+++ b/test/core/iomgr/tcp_client_posix_test.c
@@ -56,7 +56,7 @@ static gpr_timespec test_deadline(void) {
 static void finish_connection() {
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   g_connections_complete++;
-  grpc_pollset_kick(&g_pollset);
+  grpc_pollset_kick(&g_pollset, NULL);
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
 }
 
@@ -111,7 +111,8 @@ void test_succeeds(void) {
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
 
   while (g_connections_complete == connections_complete_before) {
-    grpc_pollset_work(&g_pollset, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5));
+    grpc_pollset_worker worker;
+    grpc_pollset_work(&g_pollset, &worker, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5));
   }
 
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
@@ -140,7 +141,8 @@ void test_fails(void) {
 
   /* wait for the connection callback to finish */
   while (g_connections_complete == connections_complete_before) {
-    grpc_pollset_work(&g_pollset, test_deadline());
+    grpc_pollset_worker worker;
+    grpc_pollset_work(&g_pollset, &worker, test_deadline());
   }
 
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
@@ -199,6 +201,7 @@ void test_times_out(void) {
                       gpr_now(connect_deadline.clock_type)) > 0) {
     int is_after_deadline =
         gpr_time_cmp(connect_deadline, gpr_now(GPR_CLOCK_MONOTONIC)) <= 0;
+    grpc_pollset_worker worker;
     if (is_after_deadline &&
         gpr_time_cmp(gpr_time_add(connect_deadline,
                                   gpr_time_from_seconds(1, GPR_TIMESPAN)),
@@ -208,7 +211,7 @@ void test_times_out(void) {
       GPR_ASSERT(g_connections_complete ==
                  connections_complete_before + is_after_deadline);
     }
-    grpc_pollset_work(&g_pollset, GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10));
+    grpc_pollset_work(&g_pollset, &worker, GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10));
   }
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
 
diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c
index 4e7fb446a7..17a85ceaec 100644
--- a/test/core/iomgr/tcp_posix_test.c
+++ b/test/core/iomgr/tcp_posix_test.c
@@ -186,7 +186,8 @@ static void read_test(ssize_t num_bytes, ssize_t slice_size) {
 
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   while (state.read_bytes < state.target_read_bytes) {
-    grpc_pollset_work(&g_pollset, deadline);
+    grpc_pollset_worker worker;
+    grpc_pollset_work(&g_pollset, &worker, deadline);
   }
   GPR_ASSERT(state.read_bytes == state.target_read_bytes);
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
@@ -222,7 +223,8 @@ static void large_read_test(ssize_t slice_size) {
 
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   while (state.read_bytes < state.target_read_bytes) {
-    grpc_pollset_work(&g_pollset, deadline);
+    grpc_pollset_worker worker;
+    grpc_pollset_work(&g_pollset, &worker, deadline);
   }
   GPR_ASSERT(state.read_bytes == state.target_read_bytes);
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
@@ -265,7 +267,7 @@ static void write_done(void *user_data /* write_socket_state */,
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   gpr_log(GPR_INFO, "Signalling write done");
   state->write_done = 1;
-  grpc_pollset_kick(&g_pollset);
+  grpc_pollset_kick(&g_pollset, NULL);
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
 }
 
@@ -281,8 +283,9 @@ void drain_socket_blocking(int fd, size_t num_bytes, size_t read_size) {
   GPR_ASSERT(fcntl(fd, F_SETFL, flags & ~O_NONBLOCK) == 0);
 
   for (;;) {
+    grpc_pollset_worker worker;
     gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
-    grpc_pollset_work(&g_pollset, GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10));
+    grpc_pollset_work(&g_pollset, &worker, GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10));
     gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
     do {
       bytes_read =
@@ -358,10 +361,11 @@ static void write_test(ssize_t num_bytes, ssize_t slice_size) {
     drain_socket_blocking(sv[0], num_bytes, num_bytes);
     gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
     for (;;) {
+      grpc_pollset_worker worker;
       if (state.write_done) {
         break;
       }
-      grpc_pollset_work(&g_pollset, deadline);
+      grpc_pollset_work(&g_pollset, &worker, deadline);
     }
     gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
   }
@@ -387,6 +391,7 @@ static void write_error_test(ssize_t num_bytes, ssize_t slice_size) {
   size_t num_blocks;
   gpr_slice *slices;
   int current_data = 0;
+  grpc_pollset_worker worker;
   gpr_timespec deadline = GRPC_TIMEOUT_SECONDS_TO_DEADLINE(20);
 
   gpr_log(GPR_INFO, "Start write error test with %d bytes, slice size %d",
@@ -417,7 +422,7 @@ static void write_error_test(ssize_t num_bytes, ssize_t slice_size) {
         if (state.write_done) {
           break;
         }
-        grpc_pollset_work(&g_pollset, deadline);
+        grpc_pollset_work(&g_pollset, &worker, deadline);
       }
       gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
       break;
diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c
index f8d0fe8217..b82d7c08b1 100644
--- a/test/core/iomgr/tcp_server_posix_test.c
+++ b/test/core/iomgr/tcp_server_posix_test.c
@@ -54,7 +54,7 @@ static void on_connect(void *arg, grpc_endpoint *tcp) {
 
   gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset));
   g_nconnects++;
-  grpc_pollset_kick(&g_pollset);
+  grpc_pollset_kick(&g_pollset, NULL);
   gpr_mu_unlock(GRPC_POLLSET_MU(&g_pollset));
 }
 
@@ -136,7 +136,8 @@ static void test_connect(int n) {
     gpr_log(GPR_DEBUG, "wait");
     while (g_nconnects == nconnects_before &&
            gpr_time_cmp(deadline, gpr_now(deadline.clock_type)) > 0) {
-      grpc_pollset_work(&g_pollset, deadline);
+      grpc_pollset_worker worker;
+      grpc_pollset_work(&g_pollset, &worker, deadline);
     }
     gpr_log(GPR_DEBUG, "wait done");
 
diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c
index 2df2f99269..990855ac6a 100644
--- a/test/core/security/oauth2_utils.c
+++ b/test/core/security/oauth2_utils.c
@@ -68,7 +68,7 @@ static void on_oauth2_response(void *user_data, grpc_credentials_md *md_elems,
   gpr_mu_lock(GRPC_POLLSET_MU(&request->pollset));
   request->is_done = 1;
   request->token = token;
-  grpc_pollset_kick(&request->pollset);
+  grpc_pollset_kick(&request->pollset, NULL);
   gpr_mu_unlock(GRPC_POLLSET_MU(&request->pollset));
 }
 
@@ -83,8 +83,11 @@ char *grpc_test_fetch_oauth2_token_with_credentials(grpc_credentials *creds) {
                                         on_oauth2_response, &request);
 
   gpr_mu_lock(GRPC_POLLSET_MU(&request.pollset));
-  while (!request.is_done)
-    grpc_pollset_work(&request.pollset, gpr_inf_future(GPR_CLOCK_MONOTONIC));
+  while (!request.is_done) {
+    grpc_pollset_worker worker;
+    grpc_pollset_work(&request.pollset, &worker,
+                      gpr_inf_future(GPR_CLOCK_MONOTONIC));
+  }
   gpr_mu_unlock(GRPC_POLLSET_MU(&request.pollset));
 
   grpc_pollset_shutdown(&request.pollset, do_nothing, NULL);
diff --git a/test/core/security/print_google_default_creds_token.c b/test/core/security/print_google_default_creds_token.c
index 0875cfb4fb..7238efbbfd 100644
--- a/test/core/security/print_google_default_creds_token.c
+++ b/test/core/security/print_google_default_creds_token.c
@@ -65,7 +65,7 @@ static void on_metadata_response(void *user_data,
   }
   gpr_mu_lock(GRPC_POLLSET_MU(&sync->pollset));
   sync->is_done = 1;
-  grpc_pollset_kick(&sync->pollset);
+  grpc_pollset_kick(&sync->pollset, NULL);
   gpr_mu_unlock(GRPC_POLLSET_MU(&sync->pollset));
 }
 
@@ -95,8 +95,11 @@ int main(int argc, char **argv) {
                                         on_metadata_response, &sync);
 
   gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset));
-  while (!sync.is_done)
-    grpc_pollset_work(&sync.pollset, gpr_inf_future(GPR_CLOCK_REALTIME));
+  while (!sync.is_done) {
+    grpc_pollset_worker worker;
+    grpc_pollset_work(&sync.pollset, &worker,
+                      gpr_inf_future(GPR_CLOCK_REALTIME));
+  }
   gpr_mu_unlock(GRPC_POLLSET_MU(&sync.pollset));
 
   grpc_credentials_release(creds);
diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c
index cb073f19c7..9b334b3c3e 100644
--- a/test/core/security/verify_jwt.c
+++ b/test/core/security/verify_jwt.c
@@ -79,7 +79,7 @@ static void on_jwt_verification_done(void *user_data,
 
   gpr_mu_lock(GRPC_POLLSET_MU(&sync->pollset));
   sync->is_done = 1;
-  grpc_pollset_kick(&sync->pollset);
+  grpc_pollset_kick(&sync->pollset, NULL);
   gpr_mu_unlock(GRPC_POLLSET_MU(&sync->pollset));
 }
 
@@ -109,8 +109,11 @@ int main(int argc, char **argv) {
                            on_jwt_verification_done, &sync);
 
   gpr_mu_lock(GRPC_POLLSET_MU(&sync.pollset));
-  while (!sync.is_done)
-    grpc_pollset_work(&sync.pollset, gpr_inf_future(GPR_CLOCK_REALTIME));
+  while (!sync.is_done) {
+    grpc_pollset_worker worker;
+    grpc_pollset_work(&sync.pollset, &worker,
+                      gpr_inf_future(GPR_CLOCK_REALTIME));
+  }
   gpr_mu_unlock(GRPC_POLLSET_MU(&sync.pollset));
 
   grpc_jwt_verifier_destroy(verifier);
diff --git a/test/core/util/reconnect_server.c b/test/core/util/reconnect_server.c
new file mode 100644
index 0000000000..2a2113338b
--- /dev/null
+++ b/test/core/util/reconnect_server.c
@@ -0,0 +1,161 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "test/core/util/reconnect_server.h"
+
+#include <arpa/inet.h>
+#include <grpc/grpc.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/host_port.h>
+#include <grpc/support/log.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/time.h>
+#include <string.h>
+#include "src/core/iomgr/endpoint.h"
+#include "src/core/iomgr/tcp_server.h"
+#include "test/core/util/port.h"
+
+static void pretty_print_backoffs(reconnect_server *server) {
+  gpr_timespec diff;
+  int i = 1;
+  double expected_backoff = 1000.0, backoff;
+  timestamp_list *head = server->head;
+  gpr_log(GPR_INFO, "reconnect server: new connection");
+  for (head = server->head; head && head->next; head = head->next, i++) {
+    diff = gpr_time_sub(head->next->timestamp, head->timestamp);
+    backoff = gpr_time_to_millis(diff);
+    gpr_log(GPR_INFO,
+            "retry %2d:backoff %6.2fs,expected backoff %6.2fs, jitter %4.2f%%",
+            i, backoff / 1000.0, expected_backoff / 1000.0,
+            (backoff - expected_backoff) * 100.0 / expected_backoff);
+    expected_backoff *= 1.6;
+    if (expected_backoff > 120 * 1000) {
+      expected_backoff = 120 * 1000;
+    }
+  }
+}
+
+static void on_connect(void *arg, grpc_endpoint *tcp) {
+  char *peer;
+  char *last_colon;
+  reconnect_server *server = (reconnect_server *)arg;
+  gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME);
+  timestamp_list *new_tail;
+  peer = grpc_endpoint_get_peer(tcp);
+  grpc_endpoint_shutdown(tcp);
+  grpc_endpoint_destroy(tcp);
+  if (peer) {
+    last_colon = strrchr(peer, ':');
+    if (server->peer == NULL) {
+      server->peer = peer;
+    } else {
+      if (last_colon == NULL) {
+        gpr_log(GPR_ERROR, "peer does not contain a ':'");
+      } else if (strncmp(server->peer, peer, last_colon - peer) != 0) {
+        gpr_log(GPR_ERROR, "mismatched peer! %s vs %s", server->peer, peer);
+      }
+      gpr_free(peer);
+    }
+  }
+  new_tail = gpr_malloc(sizeof(timestamp_list));
+  new_tail->timestamp = now;
+  new_tail->next = NULL;
+  if (server->tail == NULL) {
+    server->head = new_tail;
+    server->tail = new_tail;
+  } else {
+    server->tail->next = new_tail;
+    server->tail = new_tail;
+  }
+  pretty_print_backoffs(server);
+}
+
+void reconnect_server_init(reconnect_server *server) {
+  grpc_init();
+  server->tcp_server = NULL;
+  grpc_pollset_init(&server->pollset);
+  server->pollsets[0] = &server->pollset;
+  server->head = NULL;
+  server->tail = NULL;
+  server->peer = NULL;
+}
+
+void reconnect_server_start(reconnect_server *server, int port) {
+  struct sockaddr_in addr;
+  int port_added;
+
+  addr.sin_family = AF_INET;
+  addr.sin_port = htons(port);
+  inet_pton(AF_INET, "0.0.0.0", &addr.sin_addr);
+
+  server->tcp_server = grpc_tcp_server_create();
+  port_added =
+      grpc_tcp_server_add_port(server->tcp_server, &addr, sizeof(addr));
+  GPR_ASSERT(port_added == port);
+
+  grpc_tcp_server_start(server->tcp_server, server->pollsets, 1, on_connect,
+                        server);
+  gpr_log(GPR_INFO, "reconnect tcp server listening on 0.0.0.0:%d", port);
+}
+
+void reconnect_server_poll(reconnect_server *server, int seconds) {
+  grpc_pollset_worker worker;
+  gpr_timespec deadline =
+      gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
+                   gpr_time_from_seconds(seconds, GPR_TIMESPAN));
+  gpr_mu_lock(GRPC_POLLSET_MU(&server->pollset));
+  grpc_pollset_work(&server->pollset, &worker, deadline);
+  gpr_mu_unlock(GRPC_POLLSET_MU(&server->pollset));
+}
+
+void reconnect_server_clear_timestamps(reconnect_server *server) {
+  timestamp_list *new_head = server->head;
+  while (server->head) {
+    new_head = server->head->next;
+    gpr_free(server->head);
+    server->head = new_head;
+  }
+  server->tail = NULL;
+  gpr_free(server->peer);
+  server->peer = NULL;
+}
+
+static void do_nothing(void *ignored) {}
+
+void reconnect_server_destroy(reconnect_server *server) {
+  grpc_tcp_server_destroy(server->tcp_server, do_nothing, NULL);
+  reconnect_server_clear_timestamps(server);
+  grpc_pollset_shutdown(&server->pollset, do_nothing, NULL);
+  grpc_pollset_destroy(&server->pollset);
+  grpc_shutdown();
+}
diff --git a/test/core/util/reconnect_server.h b/test/core/util/reconnect_server.h
new file mode 100644
index 0000000000..8a278e0c7a
--- /dev/null
+++ b/test/core/util/reconnect_server.h
@@ -0,0 +1,69 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_TEST_CORE_UTIL_RECONNECT_SERVER_H
+#define GRPC_TEST_CORE_UTIL_RECONNECT_SERVER_H
+
+#include <grpc/support/sync.h>
+#include <grpc/support/time.h>
+#include "src/core/iomgr/tcp_server.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct timestamp_list {
+  gpr_timespec timestamp;
+  struct timestamp_list *next;
+} timestamp_list;
+
+typedef struct reconnect_server {
+  grpc_tcp_server *tcp_server;
+  grpc_pollset pollset;
+  grpc_pollset *pollsets[1];
+  timestamp_list *head;
+  timestamp_list *tail;
+  char *peer;
+} reconnect_server;
+
+void reconnect_server_init(reconnect_server *server);
+void reconnect_server_start(reconnect_server *server, int port);
+void reconnect_server_poll(reconnect_server *server, int seconds);
+void reconnect_server_destroy(reconnect_server *server);
+void reconnect_server_clear_timestamps(reconnect_server *server);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* GRPC_TEST_CORE_UTIL_RECONNECT_SERVER_H */
diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc
index b95bdf6b9b..9b53bdc999 100644
--- a/test/cpp/end2end/async_end2end_test.cc
+++ b/test/cpp/end2end/async_end2end_test.cc
@@ -592,6 +592,80 @@ TEST_F(AsyncEnd2endTest, MetadataRpc) {
   EXPECT_EQ(meta6.second, server_trailing_metadata.find(meta6.first)->second);
   EXPECT_GE(server_trailing_metadata.size(), static_cast<size_t>(2));
 }
+
+// Server uses AsyncNotifyWhenDone API to check for cancellation
+TEST_F(AsyncEnd2endTest, ServerCheckCancellation) {
+  ResetStub();
+
+  EchoRequest send_request;
+  EchoRequest recv_request;
+  EchoResponse send_response;
+  EchoResponse recv_response;
+  Status recv_status;
+
+  ClientContext cli_ctx;
+  ServerContext srv_ctx;
+  grpc::ServerAsyncResponseWriter<EchoResponse> response_writer(&srv_ctx);
+
+  send_request.set_message("Hello");
+  std::unique_ptr<ClientAsyncResponseReader<EchoResponse> > response_reader(
+      stub_->AsyncEcho(&cli_ctx, send_request, cq_.get()));
+
+  srv_ctx.AsyncNotifyWhenDone(tag(5));
+  service_.RequestEcho(&srv_ctx, &recv_request, &response_writer, cq_.get(),
+                       cq_.get(), tag(2));
+
+  Verifier().Expect(2, true).Verify(cq_.get());
+  EXPECT_EQ(send_request.message(), recv_request.message());
+
+  cli_ctx.TryCancel();
+  Verifier().Expect(5, true).Verify(cq_.get());
+  EXPECT_TRUE(srv_ctx.IsCancelled());
+
+  response_reader->Finish(&recv_response, &recv_status, tag(4));
+  Verifier().Expect(4, false).Verify(cq_.get());
+
+  EXPECT_EQ(StatusCode::CANCELLED, recv_status.error_code());
+}
+
+// Server uses AsyncNotifyWhenDone API to check for normal finish
+TEST_F(AsyncEnd2endTest, ServerCheckDone) {
+  ResetStub();
+
+  EchoRequest send_request;
+  EchoRequest recv_request;
+  EchoResponse send_response;
+  EchoResponse recv_response;
+  Status recv_status;
+
+  ClientContext cli_ctx;
+  ServerContext srv_ctx;
+  grpc::ServerAsyncResponseWriter<EchoResponse> response_writer(&srv_ctx);
+
+  send_request.set_message("Hello");
+  std::unique_ptr<ClientAsyncResponseReader<EchoResponse> > response_reader(
+      stub_->AsyncEcho(&cli_ctx, send_request, cq_.get()));
+
+  srv_ctx.AsyncNotifyWhenDone(tag(5));
+  service_.RequestEcho(&srv_ctx, &recv_request, &response_writer, cq_.get(),
+                       cq_.get(), tag(2));
+
+  Verifier().Expect(2, true).Verify(cq_.get());
+  EXPECT_EQ(send_request.message(), recv_request.message());
+
+  send_response.set_message(recv_request.message());
+  response_writer.Finish(send_response, Status::OK, tag(3));
+  Verifier().Expect(3, true).Verify(cq_.get());
+  Verifier().Expect(5, true).Verify(cq_.get());
+  EXPECT_FALSE(srv_ctx.IsCancelled());
+
+  response_reader->Finish(&recv_response, &recv_status, tag(4));
+  Verifier().Expect(4, true).Verify(cq_.get());
+
+  EXPECT_EQ(send_response.message(), recv_response.message());
+  EXPECT_TRUE(recv_status.ok());
+}
+
 }  // namespace
 }  // namespace testing
 }  // namespace grpc
diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc
index f39c6cf82a..3144ca4dc7 100644
--- a/test/cpp/end2end/end2end_test.cc
+++ b/test/cpp/end2end/end2end_test.cc
@@ -830,6 +830,31 @@ TEST_F(End2endTest, HugeResponse) {
   EXPECT_TRUE(s.ok());
 }
 
+namespace {
+void ReaderThreadFunc(ClientReaderWriter<EchoRequest, EchoResponse>* stream, gpr_event *ev) {
+  EchoResponse resp;
+  gpr_event_set(ev, (void*)1);
+  while (stream->Read(&resp)) {
+    gpr_log(GPR_INFO, "Read message");
+  }
+}
+}  // namespace
+
+// Run a Read and a WritesDone simultaneously.
+TEST_F(End2endTest, SimultaneousReadWritesDone) {
+  ResetStub();
+  ClientContext context;
+  gpr_event ev;
+  gpr_event_init(&ev);
+  auto stream = stub_->BidiStream(&context);
+  std::thread reader_thread(ReaderThreadFunc, stream.get(), &ev);
+  gpr_event_wait(&ev, gpr_inf_future(GPR_CLOCK_REALTIME));
+  stream->WritesDone();
+  Status s = stream->Finish();
+  EXPECT_TRUE(s.ok());
+  reader_thread.join();
+}
+
 TEST_F(End2endTest, Peer) {
   ResetStub();
   EchoRequest request;
diff --git a/test/cpp/interop/client.cc b/test/cpp/interop/client.cc
index d0393fafb2..ebc5cfc85a 100644
--- a/test/cpp/interop/client.cc
+++ b/test/cpp/interop/client.cc
@@ -70,6 +70,7 @@ DEFINE_string(test_case, "large_unary",
               "jwt_token_creds: large_unary with JWT token auth; "
               "oauth2_auth_token: raw oauth2 access token auth; "
               "per_rpc_creds: raw oauth2 access token on a single rpc; "
+	      "status_code_and_message: verify status code & message; "
               "all : all of above.");
 DEFINE_string(default_service_account, "",
               "Email of GCE default service account");
@@ -82,7 +83,7 @@ using grpc::testing::GetServiceAccountJsonKey;
 
 int main(int argc, char** argv) {
   grpc::testing::InitTest(&argc, &argv, true);
-
+  gpr_log(GPR_INFO, "Testing these cases: %s", FLAGS_test_case.c_str());
   int ret = 0;
   grpc::testing::InteropClient client(
       CreateChannelForTestCase(FLAGS_test_case));
@@ -121,6 +122,8 @@ int main(int argc, char** argv) {
   } else if (FLAGS_test_case == "per_rpc_creds") {
     grpc::string json_key = GetServiceAccountJsonKey();
     client.DoPerRpcCreds(json_key, FLAGS_oauth_scope);
+  } else if (FLAGS_test_case == "status_code_and_message") {
+    client.DoStatusWithMessage();
   } else if (FLAGS_test_case == "all") {
     client.DoEmpty();
     client.DoLargeUnary();
@@ -131,6 +134,7 @@ int main(int argc, char** argv) {
     client.DoCancelAfterBegin();
     client.DoCancelAfterFirstResponse();
     client.DoTimeoutOnSleepingServer();
+    client.DoStatusWithMessage();
     // service_account_creds and jwt_token_creds can only run with ssl.
     if (FLAGS_enable_ssl) {
       grpc::string json_key = GetServiceAccountJsonKey();
diff --git a/test/cpp/interop/interop_client.cc b/test/cpp/interop/interop_client.cc
index e5c0e4631f..dfb90fadc2 100644
--- a/test/cpp/interop/interop_client.cc
+++ b/test/cpp/interop/interop_client.cc
@@ -423,5 +423,24 @@ void InteropClient::DoTimeoutOnSleepingServer() {
   gpr_log(GPR_INFO, "Pingpong streaming timeout done.");
 }
 
+void InteropClient::DoStatusWithMessage() {
+  gpr_log(GPR_INFO, "Sending RPC with a request for status code 2 and message");
+  std::unique_ptr<TestService::Stub> stub(TestService::NewStub(channel_));
+
+  ClientContext context;
+  SimpleRequest request;
+  SimpleResponse response;
+  EchoStatus *requested_status = request.mutable_response_status();
+  requested_status->set_code(grpc::StatusCode::UNKNOWN);
+  grpc::string test_msg = "This is a test message";
+  requested_status->set_message(test_msg);
+
+  Status s = stub->UnaryCall(&context, request, &response);
+
+  GPR_ASSERT(s.error_code() == grpc::StatusCode::UNKNOWN);
+  GPR_ASSERT(s.error_message() == test_msg);
+  gpr_log(GPR_INFO, "Done testing Status and Message");
+}
+
 }  // namespace testing
 }  // namespace grpc
diff --git a/test/cpp/interop/interop_client.h b/test/cpp/interop/interop_client.h
index bf8188325e..6e26c49e5d 100644
--- a/test/cpp/interop/interop_client.h
+++ b/test/cpp/interop/interop_client.h
@@ -60,6 +60,7 @@ class InteropClient {
   void DoCancelAfterBegin();
   void DoCancelAfterFirstResponse();
   void DoTimeoutOnSleepingServer();
+  void DoStatusWithMessage();
   // Auth tests.
   // username is a string containing the user email
   void DoJwtTokenCreds(const grpc::string& username);
diff --git a/test/cpp/interop/reconnect_interop_client.cc b/test/cpp/interop/reconnect_interop_client.cc
new file mode 100644
index 0000000000..65f098050e
--- /dev/null
+++ b/test/cpp/interop/reconnect_interop_client.cc
@@ -0,0 +1,103 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <memory>
+#include <sstream>
+
+#include <grpc/grpc.h>
+#include <grpc/support/log.h>
+#include <gflags/gflags.h>
+#include <grpc++/channel_interface.h>
+#include <grpc++/client_context.h>
+#include <grpc++/status.h>
+#include "test/cpp/util/create_test_channel.h"
+#include "test/cpp/util/test_config.h"
+#include "test/proto/test.grpc.pb.h"
+#include "test/proto/empty.grpc.pb.h"
+#include "test/proto/messages.grpc.pb.h"
+
+DEFINE_int32(server_control_port, 0, "Server port for control rpcs.");
+DEFINE_int32(server_retry_port, 0, "Server port for testing reconnection.");
+DEFINE_string(server_host, "127.0.0.1", "Server host to connect to");
+
+using grpc::ChannelInterface;
+using grpc::ClientContext;
+using grpc::CreateTestChannel;
+using grpc::Status;
+using grpc::testing::Empty;
+using grpc::testing::ReconnectInfo;
+using grpc::testing::ReconnectService;
+
+int main(int argc, char** argv) {
+  grpc::testing::InitTest(&argc, &argv, true);
+  GPR_ASSERT(FLAGS_server_control_port);
+  GPR_ASSERT(FLAGS_server_retry_port);
+
+  std::ostringstream server_address;
+  server_address << FLAGS_server_host << ':' << FLAGS_server_control_port;
+  std::unique_ptr<ReconnectService::Stub> control_stub(
+      ReconnectService::NewStub(
+          CreateTestChannel(server_address.str(), false)));
+  ClientContext start_context;
+  Empty empty_request;
+  Empty empty_response;
+  Status start_status =
+      control_stub->Start(&start_context, empty_request, &empty_response);
+  GPR_ASSERT(start_status.ok());
+
+  gpr_log(GPR_INFO, "Starting connections with retries.");
+  server_address.str("");
+  server_address << FLAGS_server_host << ':' << FLAGS_server_retry_port;
+  std::shared_ptr<ChannelInterface> retry_channel =
+      CreateTestChannel(server_address.str(), true);
+  // About 13 retries.
+  const int kDeadlineSeconds = 540;
+  // Use any rpc to test retry.
+  std::unique_ptr<ReconnectService::Stub> retry_stub(
+      ReconnectService::NewStub(retry_channel));
+  ClientContext retry_context;
+  retry_context.set_deadline(std::chrono::system_clock::now() +
+                             std::chrono::seconds(kDeadlineSeconds));
+  Status retry_status =
+      retry_stub->Start(&retry_context, empty_request, &empty_response);
+  GPR_ASSERT(retry_status.error_code() == grpc::StatusCode::DEADLINE_EXCEEDED);
+  gpr_log(GPR_INFO, "Done retrying, getting final data from server");
+
+  ClientContext stop_context;
+  ReconnectInfo response;
+  Status stop_status =
+      control_stub->Stop(&stop_context, empty_request, &response);
+  GPR_ASSERT(stop_status.ok());
+  GPR_ASSERT(response.passed() == true);
+  return 0;
+}
diff --git a/test/cpp/interop/reconnect_interop_server.cc b/test/cpp/interop/reconnect_interop_server.cc
new file mode 100644
index 0000000000..8bc51aa52e
--- /dev/null
+++ b/test/cpp/interop/reconnect_interop_server.cc
@@ -0,0 +1,190 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <condition_variable>
+#include <memory>
+#include <mutex>
+#include <sstream>
+
+#include <signal.h>
+#include <unistd.h>
+
+#include <gflags/gflags.h>
+#include <grpc/grpc.h>
+#include <grpc/support/log.h>
+#include <grpc++/config.h>
+#include <grpc++/server.h>
+#include <grpc++/server_builder.h>
+#include <grpc++/server_context.h>
+#include <grpc++/server_credentials.h>
+#include <grpc++/status.h>
+#include "test/core/util/reconnect_server.h"
+#include "test/cpp/util/test_config.h"
+#include "test/proto/test.grpc.pb.h"
+#include "test/proto/empty.grpc.pb.h"
+#include "test/proto/messages.grpc.pb.h"
+
+DEFINE_int32(control_port, 0, "Server port for controlling the server.");
+DEFINE_int32(retry_port, 0,
+             "Server port for raw tcp connections. All incoming "
+             "connections will be closed immediately.");
+
+using grpc::Server;
+using grpc::ServerBuilder;
+using grpc::ServerContext;
+using grpc::ServerCredentials;
+using grpc::ServerReader;
+using grpc::ServerReaderWriter;
+using grpc::ServerWriter;
+using grpc::SslServerCredentialsOptions;
+using grpc::Status;
+using grpc::testing::Empty;
+using grpc::testing::ReconnectService;
+using grpc::testing::ReconnectInfo;
+
+static bool got_sigint = false;
+
+class ReconnectServiceImpl : public ReconnectService::Service {
+ public:
+  explicit ReconnectServiceImpl(int retry_port)
+      : retry_port_(retry_port), serving_(false), shutdown_(false) {
+    reconnect_server_init(&tcp_server_);
+  }
+
+  ~ReconnectServiceImpl() {
+    if (tcp_server_.tcp_server) {
+      reconnect_server_destroy(&tcp_server_);
+    }
+  }
+
+  void Poll(int seconds) { reconnect_server_poll(&tcp_server_, seconds); }
+
+  Status Start(ServerContext* context, const Empty* request, Empty* response) {
+    std::unique_lock<std::mutex> lock(mu_);
+    while (serving_ && !shutdown_) {
+      cv_.wait(lock);
+    }
+    if (shutdown_) {
+      return Status(grpc::StatusCode::UNAVAILABLE, "shutting down");
+    }
+    serving_ = true;
+    lock.unlock();
+
+    if (!tcp_server_.tcp_server) {
+      reconnect_server_start(&tcp_server_, retry_port_);
+    } else {
+      reconnect_server_clear_timestamps(&tcp_server_);
+    }
+    return Status::OK;
+  }
+
+  Status Stop(ServerContext* context, const Empty* request,
+              ReconnectInfo* response) {
+    // extract timestamps and set response
+    Verify(response);
+    reconnect_server_clear_timestamps(&tcp_server_);
+    std::lock_guard<std::mutex> lock(mu_);
+    serving_ = false;
+    cv_.notify_one();
+    return Status::OK;
+  }
+
+  void Verify(ReconnectInfo* response) {
+    double expected_backoff = 1000.0;
+    const double kTransmissionDelay = 100.0;
+    const double kBackoffMultiplier = 1.6;
+    const double kJitterFactor = 0.2;
+    const int kMaxBackoffMs = 120 * 1000;
+    bool passed = true;
+    for (timestamp_list* cur = tcp_server_.head; cur && cur->next;
+         cur = cur->next) {
+      double backoff = gpr_time_to_millis(
+          gpr_time_sub(cur->next->timestamp, cur->timestamp));
+      double min_backoff = expected_backoff * (1 - kJitterFactor);
+      double max_backoff = expected_backoff * (1 + kJitterFactor);
+      if (backoff < min_backoff - kTransmissionDelay ||
+          backoff > max_backoff + kTransmissionDelay) {
+        passed = false;
+      }
+      response->add_backoff_ms(static_cast<gpr_int32>(backoff));
+      expected_backoff *= kBackoffMultiplier;
+      expected_backoff =
+          expected_backoff > kMaxBackoffMs ? kMaxBackoffMs : expected_backoff;
+    }
+    response->set_passed(passed);
+  }
+
+  void Shutdown() {
+    std::lock_guard<std::mutex> lock(mu_);
+    shutdown_ = true;
+    cv_.notify_all();
+  }
+
+ private:
+  int retry_port_;
+  reconnect_server tcp_server_;
+  bool serving_;
+  bool shutdown_;
+  std::mutex mu_;
+  std::condition_variable cv_;
+};
+
+void RunServer() {
+  std::ostringstream server_address;
+  server_address << "0.0.0.0:" << FLAGS_control_port;
+  ReconnectServiceImpl service(FLAGS_retry_port);
+
+  ServerBuilder builder;
+  builder.RegisterService(&service);
+  builder.AddListeningPort(server_address.str(),
+                           grpc::InsecureServerCredentials());
+  std::unique_ptr<Server> server(builder.BuildAndStart());
+  gpr_log(GPR_INFO, "Server listening on %s", server_address.str().c_str());
+  while (!got_sigint) {
+    service.Poll(5);
+  }
+  service.Shutdown();
+}
+
+static void sigint_handler(int x) { got_sigint = true; }
+
+int main(int argc, char** argv) {
+  grpc::testing::InitTest(&argc, &argv, true);
+  signal(SIGINT, sigint_handler);
+
+  GPR_ASSERT(FLAGS_control_port != 0);
+  GPR_ASSERT(FLAGS_retry_port != 0);
+  RunServer();
+
+  return 0;
+}
diff --git a/test/cpp/interop/server.cc b/test/cpp/interop/server.cc
index db87872cf5..05a10de51e 100644
--- a/test/cpp/interop/server.cc
+++ b/test/cpp/interop/server.cc
@@ -105,6 +105,13 @@ class TestServiceImpl : public TestService::Service {
         return Status(grpc::StatusCode::INTERNAL, "Error creating payload.");
       }
     }
+
+    if (request->has_response_status()) {
+      return Status(static_cast<grpc::StatusCode>
+		    (request->response_status().code()),
+		    request->response_status().message()); 
+    }
+
     return Status::OK;
   }
 
diff --git a/test/proto/messages.proto b/test/proto/messages.proto
index 500e79cc81..89e55443b5 100644
--- a/test/proto/messages.proto
+++ b/test/proto/messages.proto
@@ -157,3 +157,11 @@ message StreamingOutputCallResponse {
   // Payload to increase response size.
   optional Payload payload = 1;
 }
+
+// For reconnect interop test only.
+// Server tells client whether its reconnects are following the spec and the
+// reconnect backoffs it saw.
+message ReconnectInfo {
+  optional bool passed = 1;
+  repeated int32 backoff_ms = 2;
+}
diff --git a/test/proto/test.proto b/test/proto/test.proto
index 1214152513..368522dc4c 100644
--- a/test/proto/test.proto
+++ b/test/proto/test.proto
@@ -79,3 +79,9 @@ service UnimplementedService {
   // A call that no server should implement
   rpc UnimplementedCall(grpc.testing.Empty) returns(grpc.testing.Empty);  
 }
+
+// A service used to control reconnect server.
+service ReconnectService {
+  rpc Start(grpc.testing.Empty) returns (grpc.testing.Empty);
+  rpc Stop(grpc.testing.Empty) returns (grpc.testing.ReconnectInfo);
+}