Merge github.com:grpc/grpc into we-are-one

2 files changed, 820 insertions, 0 deletions

diff --git a/test/core/client_config/lb_policies_test.c b/test/core/client_config/lb_policies_test.c
new file mode 100644
index 0000000000..ee3164f4fd
--- /dev/null
+++ b/test/core/client_config/lb_policies_test.c
@@ -0,0 +1,705 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <stdarg.h>
+#include <string.h>
+
+#include <grpc/grpc.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/host_port.h>
+#include <grpc/support/log.h>
+#include <grpc/support/time.h>
+#include <grpc/support/string_util.h>
+
+#include "src/core/channel/channel_stack.h"
+#include "src/core/surface/channel.h"
+#include "src/core/channel/client_channel.h"
+#include "src/core/support/string.h"
+#include "src/core/surface/server.h"
+#include "test/core/util/test_config.h"
+#include "test/core/util/port.h"
+#include "test/core/end2end/cq_verifier.h"
+
+typedef struct servers_fixture {
+  size_t num_servers;
+  grpc_server **servers;
+  grpc_call **server_calls;
+  grpc_completion_queue *cq;
+  char **servers_hostports;
+  grpc_metadata_array *request_metadata_recv;
+} servers_fixture;
+
+typedef void (*verifier_fn)(const servers_fixture *, grpc_channel *,
+                            const int *, const size_t);
+
+typedef struct test_spec {
+  size_t num_iters;
+  size_t num_servers;
+
+  int **kill_at;
+  int **revive_at;
+
+  const char *description;
+
+  verifier_fn verifier;
+
+} test_spec;
+
+static void test_spec_reset(test_spec *spec) {
+  size_t i, j;
+
+  for (i = 0; i < spec->num_iters; i++) {
+    for (j = 0; j < spec->num_servers; j++) {
+      spec->kill_at[i][j] = 0;
+      spec->revive_at[i][j] = 0;
+    }
+  }
+}
+
+static test_spec *test_spec_create(size_t num_iters, size_t num_servers) {
+  test_spec *spec;
+  size_t i;
+
+  spec = gpr_malloc(sizeof(test_spec));
+  spec->num_iters = num_iters;
+  spec->num_servers = num_servers;
+  spec->kill_at = gpr_malloc(sizeof(int *) * num_iters);
+  spec->revive_at = gpr_malloc(sizeof(int *) * num_iters);
+  for (i = 0; i < num_iters; i++) {
+    spec->kill_at[i] = gpr_malloc(sizeof(int) * num_servers);
+    spec->revive_at[i] = gpr_malloc(sizeof(int) * num_servers);
+  }
+
+  test_spec_reset(spec);
+  return spec;
+}
+
+static void test_spec_destroy(test_spec *spec) {
+  size_t i;
+  for (i = 0; i < spec->num_iters; i++) {
+    gpr_free(spec->kill_at[i]);
+    gpr_free(spec->revive_at[i]);
+  }
+
+  gpr_free(spec->kill_at);
+  gpr_free(spec->revive_at);
+
+  gpr_free(spec);
+}
+
+static void *tag(gpr_intptr t) { return (void *)t; }
+
+static gpr_timespec n_seconds_time(int n) {
+  return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(n);
+}
+
+static void drain_cq(grpc_completion_queue *cq) {
+  grpc_event ev;
+  do {
+    ev = grpc_completion_queue_next(cq, n_seconds_time(5), NULL);
+  } while (ev.type != GRPC_QUEUE_SHUTDOWN);
+}
+
+static void kill_server(const servers_fixture *f, size_t i) {
+  gpr_log(GPR_INFO, "KILLING SERVER %d", i);
+  GPR_ASSERT(f->servers[i] != NULL);
+  grpc_server_shutdown_and_notify(f->servers[i], f->cq, tag(10000));
+  GPR_ASSERT(grpc_completion_queue_pluck(
+                 f->cq, tag(10000), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5), NULL)
+                 .type == GRPC_OP_COMPLETE);
+  grpc_server_destroy(f->servers[i]);
+  f->servers[i] = NULL;
+}
+
+static void revive_server(const servers_fixture *f, size_t i) {
+  int got_port;
+  gpr_log(GPR_INFO, "RAISE AGAIN SERVER %d", i);
+  GPR_ASSERT(f->servers[i] == NULL);
+  f->servers[i] = grpc_server_create(NULL, NULL);
+  grpc_server_register_completion_queue(f->servers[i], f->cq, NULL);
+  GPR_ASSERT((got_port = grpc_server_add_insecure_http2_port(
+                  f->servers[i], f->servers_hostports[i])) > 0);
+  grpc_server_start(f->servers[i]);
+}
+
+static servers_fixture *setup_servers(const char *server_host,
+                                      const size_t num_servers) {
+  servers_fixture *f = gpr_malloc(sizeof(servers_fixture));
+  int *ports;
+  int got_port;
+  size_t i;
+
+  f->num_servers = num_servers;
+  f->server_calls = gpr_malloc(sizeof(grpc_call *) * num_servers);
+  f->request_metadata_recv =
+      gpr_malloc(sizeof(grpc_metadata_array) * num_servers);
+  /* Create servers. */
+  ports = gpr_malloc(sizeof(int *) * num_servers);
+  f->servers = gpr_malloc(sizeof(grpc_server *) * num_servers);
+  f->servers_hostports = gpr_malloc(sizeof(char *) * num_servers);
+  f->cq = grpc_completion_queue_create(NULL);
+  for (i = 0; i < num_servers; i++) {
+    ports[i] = grpc_pick_unused_port_or_die();
+
+    gpr_join_host_port(&f->servers_hostports[i], server_host, ports[i]);
+
+    f->servers[i] = grpc_server_create(NULL, NULL);
+    grpc_server_register_completion_queue(f->servers[i], f->cq, NULL);
+    GPR_ASSERT((got_port = grpc_server_add_insecure_http2_port(
+                    f->servers[i], f->servers_hostports[i])) > 0);
+    GPR_ASSERT(ports[i] == got_port);
+    grpc_server_start(f->servers[i]);
+  }
+  gpr_free(ports);
+  return f;
+}
+
+static void teardown_servers(servers_fixture *f) {
+  size_t i;
+  /* Destroy server. */
+  for (i = 0; i < f->num_servers; i++) {
+    if (f->servers[i] == NULL) continue;
+    grpc_server_shutdown_and_notify(f->servers[i], f->cq, tag(10000));
+    GPR_ASSERT(grpc_completion_queue_pluck(
+                   f->cq, tag(10000), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5), NULL)
+                   .type == GRPC_OP_COMPLETE);
+    grpc_server_destroy(f->servers[i]);
+  }
+  grpc_completion_queue_shutdown(f->cq);
+  drain_cq(f->cq);
+  grpc_completion_queue_destroy(f->cq);
+
+  gpr_free(f->servers);
+
+  for (i = 0; i < f->num_servers; i++) {
+    gpr_free(f->servers_hostports[i]);
+  }
+
+  gpr_free(f->servers_hostports);
+  gpr_free(f->request_metadata_recv);
+  gpr_free(f->server_calls);
+  gpr_free(f);
+}
+
+/** Returns connection sequence (server indices), which must be freed */
+int *perform_request(servers_fixture *f, grpc_channel *client,
+                     const test_spec *spec) {
+  grpc_call *c;
+  int s_idx;
+  int *s_valid;
+  gpr_timespec deadline;
+  grpc_op ops[6];
+  grpc_op *op;
+  grpc_status_code status;
+  char *details;
+  size_t details_capacity;
+  int was_cancelled;
+  grpc_call_details *call_details;
+  size_t i, iter_num;
+  grpc_event ev;
+  int read_tag;
+  int *connection_sequence;
+  grpc_metadata_array initial_metadata_recv;
+  grpc_metadata_array trailing_metadata_recv;
+
+  s_valid = gpr_malloc(sizeof(int) * f->num_servers);
+  call_details = gpr_malloc(sizeof(grpc_call_details) * f->num_servers);
+  connection_sequence = gpr_malloc(sizeof(int) * spec->num_iters);
+
+  /* Send a trivial request. */
+  deadline = n_seconds_time(60);
+
+  for (iter_num = 0; iter_num < spec->num_iters; iter_num++) {
+    cq_verifier *cqv = cq_verifier_create(f->cq);
+    details = NULL;
+    details_capacity = 0;
+    was_cancelled = 2;
+
+    for (i = 0; i < f->num_servers; i++) {
+      if (spec->kill_at[iter_num][i] != 0) {
+        kill_server(f, i);
+      } else if (spec->revive_at[iter_num][i] != 0) {
+        /* killing takes precedence */
+        revive_server(f, i);
+      }
+    }
+
+    connection_sequence[iter_num] = -1;
+    grpc_metadata_array_init(&initial_metadata_recv);
+    grpc_metadata_array_init(&trailing_metadata_recv);
+
+    for (i = 0; i < f->num_servers; i++) {
+      grpc_call_details_init(&call_details[i]);
+    }
+    memset(s_valid, 0, f->num_servers * sizeof(int));
+
+    c = grpc_channel_create_call(client, NULL, GRPC_PROPAGATE_DEFAULTS, f->cq,
+                                 "/foo", "foo.test.google.fr", deadline, NULL);
+    GPR_ASSERT(c);
+
+    op = ops;
+    op->op = GRPC_OP_SEND_INITIAL_METADATA;
+    op->data.send_initial_metadata.count = 0;
+    op->flags = 0;
+    op->reserved = NULL;
+    op++;
+    op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
+    op->flags = 0;
+    op->reserved = NULL;
+    op++;
+    op->op = GRPC_OP_RECV_INITIAL_METADATA;
+    op->data.recv_initial_metadata = &initial_metadata_recv;
+    op->flags = 0;
+    op->reserved = NULL;
+    op++;
+    op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
+    op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
+    op->data.recv_status_on_client.status = &status;
+    op->data.recv_status_on_client.status_details = &details;
+    op->data.recv_status_on_client.status_details_capacity = &details_capacity;
+    op->flags = 0;
+    op->reserved = NULL;
+    op++;
+    GPR_ASSERT(GRPC_CALL_OK ==
+               grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL));
+
+    /* "listen" on all servers */
+    for (i = 0; i < f->num_servers; i++) {
+      grpc_metadata_array_init(&f->request_metadata_recv[i]);
+      if (f->servers[i] != NULL) {
+        GPR_ASSERT(GRPC_CALL_OK ==
+                   grpc_server_request_call(f->servers[i], &f->server_calls[i],
+                                            &call_details[i],
+                                            &f->request_metadata_recv[i], f->cq,
+                                            f->cq, tag(1000 + (int)i)));
+      }
+    }
+
+    s_idx = -1;
+    while ((ev = grpc_completion_queue_next(
+                f->cq, GRPC_TIMEOUT_SECONDS_TO_DEADLINE(1), NULL))
+               .type != GRPC_QUEUE_TIMEOUT) {
+      read_tag = ((int)(gpr_intptr)ev.tag);
+      gpr_log(GPR_DEBUG, "EVENT: success:%d, type:%d, tag:%d iter:%d",
+              ev.success, ev.type, read_tag, iter_num);
+      if (ev.success && read_tag >= 1000) {
+        GPR_ASSERT(s_idx == -1); /* only one server must reply */
+        /* only server notifications for non-shutdown events */
+        s_idx = read_tag - 1000;
+        s_valid[s_idx] = 1;
+        connection_sequence[iter_num] = s_idx;
+      }
+    }
+
+    if (s_idx >= 0) {
+      op = ops;
+      op->op = GRPC_OP_SEND_INITIAL_METADATA;
+      op->data.send_initial_metadata.count = 0;
+      op->flags = 0;
+      op->reserved = NULL;
+      op++;
+      op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
+      op->data.send_status_from_server.trailing_metadata_count = 0;
+      op->data.send_status_from_server.status = GRPC_STATUS_UNIMPLEMENTED;
+      op->data.send_status_from_server.status_details = "xyz";
+      op->flags = 0;
+      op->reserved = NULL;
+      op++;
+      op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
+      op->data.recv_close_on_server.cancelled = &was_cancelled;
+      op->flags = 0;
+      op->reserved = NULL;
+      op++;
+      GPR_ASSERT(GRPC_CALL_OK == grpc_call_start_batch(f->server_calls[s_idx],
+                                                       ops, (size_t)(op - ops),
+                                                       tag(102), NULL));
+
+      cq_expect_completion(cqv, tag(102), 1);
+      cq_expect_completion(cqv, tag(1), 1);
+      cq_verify(cqv);
+
+      GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED);
+      GPR_ASSERT(0 == strcmp(details, "xyz"));
+      GPR_ASSERT(0 == strcmp(call_details[s_idx].method, "/foo"));
+      GPR_ASSERT(0 == strcmp(call_details[s_idx].host, "foo.test.google.fr"));
+      GPR_ASSERT(was_cancelled == 1);
+    }
+
+    for (i = 0; i < f->num_servers; i++) {
+      if (s_valid[i] != 0) {
+        grpc_call_destroy(f->server_calls[i]);
+      }
+      grpc_metadata_array_destroy(&f->request_metadata_recv[i]);
+    }
+    grpc_metadata_array_destroy(&initial_metadata_recv);
+    grpc_metadata_array_destroy(&trailing_metadata_recv);
+
+    cq_verifier_destroy(cqv);
+
+    grpc_call_destroy(c);
+
+    for (i = 0; i < f->num_servers; i++) {
+      grpc_call_details_destroy(&call_details[i]);
+    }
+    gpr_free(details);
+  }
+
+  gpr_free(call_details);
+  gpr_free(s_valid);
+
+  return connection_sequence;
+}
+
+static void assert_channel_connectivity(
+    grpc_channel *ch, size_t num_accepted_conn_states,
+    grpc_connectivity_state accepted_conn_state, ...) {
+  size_t i;
+  grpc_channel_stack *client_stack;
+  grpc_channel_element *client_channel_filter;
+  grpc_connectivity_state actual_conn_state;
+  grpc_call_list call_list = GRPC_CALL_LIST_INIT;
+  va_list ap;
+
+  client_stack = grpc_channel_get_channel_stack(ch);
+  client_channel_filter = grpc_channel_stack_last_element(client_stack);
+
+  actual_conn_state = grpc_client_channel_check_connectivity_state(
+      client_channel_filter, 0 /* don't try to connect */, &call_list);
+  grpc_call_list_run(&call_list);
+  va_start(ap, accepted_conn_state);
+  for (i = 0; i < num_accepted_conn_states; i++) {
+    if (actual_conn_state == accepted_conn_state) {
+      break;
+    }
+    accepted_conn_state = va_arg(ap, grpc_connectivity_state);
+  }
+  va_end(ap);
+  if (i == num_accepted_conn_states) {
+    char **accepted_strs =
+        gpr_malloc(sizeof(char *) * num_accepted_conn_states);
+    char *accepted_str_joined;
+    va_start(ap, accepted_conn_state);
+    for (i = 0; i < num_accepted_conn_states; i++) {
+      GPR_ASSERT(gpr_asprintf(&accepted_strs[i], "%d", accepted_conn_state) >
+                 0);
+      accepted_conn_state = va_arg(ap, grpc_connectivity_state);
+    }
+    va_end(ap);
+    accepted_str_joined = gpr_strjoin_sep((const char **)accepted_strs,
+                                          num_accepted_conn_states, ", ", NULL);
+    gpr_log(
+        GPR_ERROR,
+        "Channel connectivity assertion failed: expected <one of [%s]>, got %d",
+        accepted_str_joined, actual_conn_state);
+
+    for (i = 0; i < num_accepted_conn_states; i++) {
+      gpr_free(accepted_strs[i]);
+    }
+    gpr_free(accepted_strs);
+    gpr_free(accepted_str_joined);
+    abort();
+  }
+}
+
+void run_spec(const test_spec *spec) {
+  grpc_channel *client;
+  char *client_hostport;
+  char *servers_hostports_str;
+  int *actual_connection_sequence;
+  servers_fixture *f = setup_servers("127.0.0.1", spec->num_servers);
+
+  /* Create client. */
+  servers_hostports_str = gpr_strjoin_sep((const char **)f->servers_hostports,
+                                          f->num_servers, ",", NULL);
+  gpr_asprintf(&client_hostport, "ipv4:%s?lb_policy=round_robin",
+               servers_hostports_str);
+  client = grpc_insecure_channel_create(client_hostport, NULL, NULL);
+
+  gpr_log(GPR_INFO, "Testing '%s' with servers=%s client=%s", spec->description,
+          servers_hostports_str, client_hostport);
+
+  actual_connection_sequence = perform_request(f, client, spec);
+
+  spec->verifier(f, client, actual_connection_sequence, spec->num_iters);
+
+  gpr_free(client_hostport);
+  gpr_free(servers_hostports_str);
+  gpr_free(actual_connection_sequence);
+
+  grpc_channel_destroy(client);
+  teardown_servers(f);
+}
+
+static void print_failed_expectations(const int *expected_connection_sequence,
+                                      const int *actual_connection_sequence,
+                                      const size_t expected_seq_length,
+                                      const size_t num_iters) {
+  size_t i;
+  for (i = 0; i < num_iters; i++) {
+    gpr_log(GPR_ERROR, "FAILURE: Iter, expected, actual:%d (%d, %d)", i,
+            expected_connection_sequence[i % expected_seq_length],
+            actual_connection_sequence[i]);
+  }
+}
+
+static void verify_vanilla_round_robin(const servers_fixture *f,
+                                       grpc_channel *client,
+                                       const int *actual_connection_sequence,
+                                       const size_t num_iters) {
+  int *expected_connection_sequence;
+  size_t i;
+  const size_t expected_seq_length = f->num_servers;
+
+  /* verify conn. seq. expectation */
+  /* get the first sequence of "num_servers" elements */
+  expected_connection_sequence = gpr_malloc(sizeof(int) * expected_seq_length);
+  memcpy(expected_connection_sequence, actual_connection_sequence,
+         sizeof(int) * expected_seq_length);
+
+  for (i = 0; i < num_iters; i++) {
+    const int actual = actual_connection_sequence[i];
+    const int expected = expected_connection_sequence[i % expected_seq_length];
+    if (actual != expected) {
+      gpr_log(GPR_ERROR, "FAILURE: expected %d, actual %d at iter %d", expected,
+              actual, i);
+      print_failed_expectations(expected_connection_sequence,
+                                actual_connection_sequence, expected_seq_length,
+                                num_iters);
+      abort();
+    }
+  }
+  assert_channel_connectivity(client, 1, GRPC_CHANNEL_READY);
+
+  gpr_free(expected_connection_sequence);
+}
+
+/* At the start of the second iteration, all but the first and last servers (as
+ * given in "f") are killed */
+static void verify_vanishing_floor_round_robin(
+    const servers_fixture *f, grpc_channel *client,
+    const int *actual_connection_sequence, const size_t num_iters) {
+  int *expected_connection_sequence;
+  const size_t expected_seq_length = 2;
+  size_t i;
+
+  /* verify conn. seq. expectation */
+  /* copy the first full sequence (without -1s) */
+  expected_connection_sequence = gpr_malloc(sizeof(int) * expected_seq_length);
+  memcpy(expected_connection_sequence, actual_connection_sequence + 2,
+         expected_seq_length * sizeof(int));
+
+  /* first three elements of the sequence should be [<1st>, -1] */
+  if (actual_connection_sequence[0] != expected_connection_sequence[0]) {
+    gpr_log(GPR_ERROR, "FAILURE: expected %d, actual %d at iter %d",
+            expected_connection_sequence[0], actual_connection_sequence[0], 0);
+    print_failed_expectations(expected_connection_sequence,
+                              actual_connection_sequence, expected_seq_length,
+                              1u);
+    abort();
+  }
+
+  GPR_ASSERT(actual_connection_sequence[1] == -1);
+
+  for (i = 2; i < num_iters; i++) {
+    const int actual = actual_connection_sequence[i];
+    const int expected = expected_connection_sequence[i % expected_seq_length];
+    if (actual != expected) {
+      gpr_log(GPR_ERROR, "FAILURE: expected %d, actual %d at iter %d", expected,
+              actual, i);
+      print_failed_expectations(expected_connection_sequence,
+                                actual_connection_sequence, expected_seq_length,
+                                num_iters);
+      abort();
+    }
+  }
+  gpr_free(expected_connection_sequence);
+}
+
+static void verify_total_carnage_round_robin(
+    const servers_fixture *f, grpc_channel *client,
+    const int *actual_connection_sequence, const size_t num_iters) {
+  size_t i;
+
+  for (i = 0; i < num_iters; i++) {
+    const int actual = actual_connection_sequence[i];
+    const int expected = -1;
+    if (actual != expected) {
+      gpr_log(GPR_ERROR, "FAILURE: expected %d, actual %d at iter %d", expected,
+              actual, i);
+      abort();
+    }
+  }
+
+  /* even though we know all the servers are dead, the client is still trying
+   * retrying, believing it's in a transient failure situation */
+  assert_channel_connectivity(client, 2, GRPC_CHANNEL_TRANSIENT_FAILURE,
+                              GRPC_CHANNEL_CONNECTING);
+}
+
+static void verify_partial_carnage_round_robin(
+    const servers_fixture *f, grpc_channel *client,
+    const int *actual_connection_sequence, const size_t num_iters) {
+  int *expected_connection_sequence;
+  size_t i;
+  const size_t expected_seq_length = f->num_servers;
+
+  /* verify conn. seq. expectation */
+  /* get the first sequence of "num_servers" elements */
+  expected_connection_sequence = gpr_malloc(sizeof(int) * expected_seq_length);
+  memcpy(expected_connection_sequence, actual_connection_sequence,
+         sizeof(int) * expected_seq_length);
+
+  for (i = 0; i < num_iters / 2; i++) {
+    const int actual = actual_connection_sequence[i];
+    const int expected = expected_connection_sequence[i % expected_seq_length];
+    if (actual != expected) {
+      gpr_log(GPR_ERROR, "FAILURE: expected %d, actual %d at iter %d", expected,
+              actual, i);
+      print_failed_expectations(expected_connection_sequence,
+                                actual_connection_sequence, expected_seq_length,
+                                num_iters);
+      abort();
+    }
+  }
+
+  /* second half of the iterations go without response */
+  for (; i < num_iters; i++) {
+    GPR_ASSERT(actual_connection_sequence[i] == -1);
+  }
+
+  /* even though we know all the servers are dead, the client is still trying
+   * retrying, believing it's in a transient failure situation */
+  assert_channel_connectivity(client, 2, GRPC_CHANNEL_TRANSIENT_FAILURE,
+                              GRPC_CHANNEL_CONNECTING);
+  gpr_free(expected_connection_sequence);
+}
+
+static void verify_rebirth_round_robin(const servers_fixture *f,
+                                       grpc_channel *client,
+                                       const int *actual_connection_sequence,
+                                       const size_t num_iters) {
+  int *expected_connection_sequence;
+  size_t i;
+  const size_t expected_seq_length = f->num_servers;
+
+  /* verify conn. seq. expectation */
+  /* get the first sequence of "num_servers" elements */
+  expected_connection_sequence = gpr_malloc(sizeof(int) * expected_seq_length);
+  memcpy(expected_connection_sequence, actual_connection_sequence + 4,
+         sizeof(int) * expected_seq_length);
+
+  /* first iteration succeeds */
+  GPR_ASSERT(actual_connection_sequence[0] != -1);
+
+  /* back up on the third (or maybe fourth) iteration */
+  i = 3;
+  if (actual_connection_sequence[i] == -1) {
+    i = 4;
+  }
+  for (; i < num_iters; i++) {
+    const int actual = actual_connection_sequence[i];
+    const int expected = expected_connection_sequence[i % expected_seq_length];
+    if (actual != expected) {
+      gpr_log(GPR_ERROR, "FAILURE: expected %d, actual %d at iter %d", expected,
+              actual, i);
+      print_failed_expectations(expected_connection_sequence,
+                                actual_connection_sequence, expected_seq_length,
+                                num_iters);
+      abort();
+    }
+  }
+
+  /* things are fine once the servers are brought back up */
+  assert_channel_connectivity(client, 1, GRPC_CHANNEL_READY);
+  gpr_free(expected_connection_sequence);
+}
+
+int main(int argc, char **argv) {
+  test_spec *spec;
+  size_t i;
+  const size_t NUM_ITERS = 10;
+  const size_t NUM_SERVERS = 4;
+
+  grpc_test_init(argc, argv);
+  grpc_init();
+
+  /* everything is fine, all servers stay up the whole time and life's peachy */
+  spec = test_spec_create(NUM_ITERS, NUM_SERVERS);
+  spec->verifier = verify_vanilla_round_robin;
+  spec->description = "test_all_server_up";
+  run_spec(spec);
+
+  /* Kill all servers first thing in the morning */
+  test_spec_reset(spec);
+  spec->verifier = verify_total_carnage_round_robin;
+  spec->description = "test_kill_all_server";
+  for (i = 0; i < NUM_SERVERS; i++) {
+    spec->kill_at[0][i] = 1;
+  }
+  run_spec(spec);
+
+  /* at the start of the 2nd iteration, kill all but the first and last servers.
+   * This should knock down the server bound to be selected next */
+  test_spec_reset(spec);
+  spec->verifier = verify_vanishing_floor_round_robin;
+  spec->description = "test_kill_all_server_at_2nd_iteration";
+  for (i = 1; i < NUM_SERVERS - 1; i++) {
+    spec->kill_at[1][i] = 1;
+  }
+  run_spec(spec);
+
+  /* Midway, kill all servers. */
+  test_spec_reset(spec);
+  spec->verifier = verify_partial_carnage_round_robin;
+  spec->description = "test_kill_all_server_midway";
+  for (i = 0; i < NUM_SERVERS; i++) {
+    spec->kill_at[spec->num_iters / 2][i] = 1;
+  }
+  run_spec(spec);
+
+  /* After first iteration, kill all servers. On the third one, bring them all
+   * back up. */
+  test_spec_reset(spec);
+  spec->verifier = verify_rebirth_round_robin;
+  spec->description = "test_kill_all_server_after_1st_resurrect_at_3rd";
+  for (i = 0; i < NUM_SERVERS; i++) {
+    spec->kill_at[1][i] = 1;
+    spec->revive_at[3][i] = 1;
+  }
+  run_spec(spec);
+
+  test_spec_destroy(spec);
+
+  grpc_shutdown();
+  return 0;
+}
diff --git a/test/core/security/credentials_test.c b/test/core/security/credentials_test.c
index 3d0ed094c9..c7bd7ab4f4 100644
--- a/test/core/security/credentials_test.c
+++ b/test/core/security/credentials_test.c
@@ -875,6 +875,119 @@ static void test_google_default_creds_access_token(void) {
   gpr_setenv(GRPC_GOOGLE_CREDENTIALS_ENV_VAR, ""); /* Reset. */
 }
 
+typedef enum {
+  PLUGIN_INITIAL_STATE,
+  PLUGIN_GET_METADATA_CALLED_STATE,
+  PLUGIN_DESTROY_CALLED_STATE
+} plugin_state;
+
+typedef struct {
+  const char *key;
+  const char *value;
+} plugin_metadata;
+
+static const plugin_metadata plugin_md[] = {{"foo", "bar"}, {"hi", "there"}};
+
+static void plugin_get_metadata_success(void *state, const char *service_url,
+                                        grpc_credentials_plugin_metadata_cb cb,
+                                        void *user_data) {
+  size_t i;
+  grpc_metadata md[GPR_ARRAY_SIZE(plugin_md)];
+  plugin_state *s = (plugin_state *)state;
+  GPR_ASSERT(strcmp(service_url, test_service_url) == 0);
+  *s = PLUGIN_GET_METADATA_CALLED_STATE;
+  for (i = 0; i < GPR_ARRAY_SIZE(plugin_md); i++) {
+    memset(&md[i], 0, sizeof(grpc_metadata));
+    md[i].key = plugin_md[i].key;
+    md[i].value = plugin_md[i].value;
+    md[i].value_length = strlen(plugin_md[i].value);
+  }
+  cb(user_data, md, GPR_ARRAY_SIZE(md), GRPC_STATUS_OK, NULL);
+}
+
+static void plugin_get_metadata_failure(void *state, const char *service_url,
+                                        grpc_credentials_plugin_metadata_cb cb,
+                                        void *user_data) {
+  plugin_state *s = (plugin_state *)state;
+  GPR_ASSERT(strcmp(service_url, test_service_url) == 0);
+  *s = PLUGIN_GET_METADATA_CALLED_STATE;
+  cb(user_data, NULL, 0, GRPC_STATUS_UNAUTHENTICATED,
+     "Could not get metadata for plugin.");
+}
+
+static void on_plugin_metadata_received_success(void *user_data,
+                                                grpc_credentials_md *md_elems,
+                                                size_t num_md,
+                                                grpc_credentials_status status,
+                                                grpc_call_list *call_list) {
+  size_t i = 0;
+  GPR_ASSERT(user_data == NULL);
+  GPR_ASSERT(md_elems != NULL);
+  GPR_ASSERT(num_md == GPR_ARRAY_SIZE(plugin_md));
+  for (i = 0; i < num_md; i++) {
+    GPR_ASSERT(gpr_slice_str_cmp(md_elems[i].key, plugin_md[i].key) == 0);
+    GPR_ASSERT(gpr_slice_str_cmp(md_elems[i].value, plugin_md[i].value) == 0);
+  }
+}
+
+static void on_plugin_metadata_received_failure(void *user_data,
+                                                grpc_credentials_md *md_elems,
+                                                size_t num_md,
+                                                grpc_credentials_status status,
+                                                grpc_call_list *call_list) {
+  GPR_ASSERT(user_data == NULL);
+  GPR_ASSERT(md_elems == NULL);
+  GPR_ASSERT(num_md == 0);
+  GPR_ASSERT(status == GRPC_CREDENTIALS_ERROR);
+}
+
+static void plugin_destroy(void *state) {
+  plugin_state *s = (plugin_state *)state;
+  *s = PLUGIN_DESTROY_CALLED_STATE;
+}
+
+static void test_metadata_plugin_success(void) {
+  grpc_credentials *creds;
+  plugin_state state = PLUGIN_INITIAL_STATE;
+  grpc_metadata_credentials_plugin plugin;
+  grpc_call_list call_list = GRPC_CALL_LIST_INIT;
+
+  plugin.state = &state;
+  plugin.get_metadata = plugin_get_metadata_success;
+  plugin.destroy = plugin_destroy;
+
+  creds = grpc_metadata_credentials_create_from_plugin(plugin, NULL);
+  GPR_ASSERT(state == PLUGIN_INITIAL_STATE);
+  grpc_credentials_get_request_metadata(creds, NULL, test_service_url,
+                                        on_plugin_metadata_received_success,
+                                        NULL, &call_list);
+  GPR_ASSERT(state == PLUGIN_GET_METADATA_CALLED_STATE);
+  grpc_credentials_release(creds);
+  GPR_ASSERT(state == PLUGIN_DESTROY_CALLED_STATE);
+  grpc_call_list_run(&call_list);
+}
+
+static void test_metadata_plugin_failure(void) {
+  grpc_credentials *creds;
+  plugin_state state = PLUGIN_INITIAL_STATE;
+  grpc_metadata_credentials_plugin plugin;
+  grpc_call_list call_list = GRPC_CALL_LIST_INIT;
+
+  plugin.state = &state;
+  plugin.get_metadata = plugin_get_metadata_failure;
+  plugin.destroy = plugin_destroy;
+
+  creds = grpc_metadata_credentials_create_from_plugin(plugin, NULL);
+  GPR_ASSERT(state == PLUGIN_INITIAL_STATE);
+  grpc_credentials_get_request_metadata(creds, NULL, test_service_url,
+                                        on_plugin_metadata_received_failure,
+                                        NULL, &call_list);
+  GPR_ASSERT(state == PLUGIN_GET_METADATA_CALLED_STATE);
+  grpc_credentials_release(creds);
+  GPR_ASSERT(state == PLUGIN_DESTROY_CALLED_STATE);
+  grpc_call_list_run(&call_list);
+}
+
 int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
   test_empty_md_store();
@@ -902,5 +1015,7 @@ int main(int argc, char **argv) {
   test_jwt_creds_signing_failure();
   test_google_default_creds_auth_key();
   test_google_default_creds_access_token();
+  test_metadata_plugin_success();
+  test_metadata_plugin_failure();
   return 0;
 }