Merge remote-tracking branch 'upstream/master' into service_config_json

13 files changed, 692 insertions, 143 deletions

diff --git a/test/core/channel/channel_args_test.c b/test/core/channel/channel_args_test.c
index 8ef1bff22e..d3eb969f09 100644
--- a/test/core/channel/channel_args_test.c
+++ b/test/core/channel/channel_args_test.c
@@ -134,12 +134,26 @@ static void test_compression_algorithm_states(void) {
   grpc_channel_args_destroy(ch_args);
 }
 
+static void test_set_socket_mutator(void) {
+  grpc_channel_args *ch_args;
+  grpc_socket_mutator mutator;
+  grpc_socket_mutator_init(&mutator, NULL);
+
+  ch_args = grpc_channel_args_set_socket_mutator(NULL, &mutator);
+  GPR_ASSERT(ch_args->num_args == 1);
+  GPR_ASSERT(strcmp(ch_args->args[0].key, GRPC_ARG_SOCKET_MUTATOR) == 0);
+  GPR_ASSERT(ch_args->args[0].type == GRPC_ARG_POINTER);
+
+  grpc_channel_args_destroy(ch_args);
+}
+
 int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
   grpc_init();
   test_create();
   test_set_compression_algorithm();
   test_compression_algorithm_states();
+  test_set_socket_mutator();
   grpc_shutdown();
   return 0;
 }
diff --git a/test/core/channel/channel_stack_test.c b/test/core/channel/channel_stack_test.c
index 1e57df9026..0840820cca 100644
--- a/test/core/channel/channel_stack_test.c
+++ b/test/core/channel/channel_stack_test.c
@@ -137,9 +137,10 @@ static void test_create_channel_stack(void) {
   GPR_ASSERT(*channel_data == 0);
 
   call_stack = gpr_malloc(channel_stack->call_stack_size);
-  grpc_error *error = grpc_call_stack_init(
-      &exec_ctx, channel_stack, 1, free_call, call_stack, NULL, NULL, path,
-      gpr_inf_future(GPR_CLOCK_MONOTONIC), call_stack);
+  grpc_error *error =
+      grpc_call_stack_init(&exec_ctx, channel_stack, 1, free_call, call_stack,
+                           NULL, NULL, path, gpr_now(GPR_CLOCK_MONOTONIC),
+                           gpr_inf_future(GPR_CLOCK_MONOTONIC), call_stack);
   GPR_ASSERT(error == GRPC_ERROR_NONE);
   GPR_ASSERT(call_stack->count == 1);
   call_elem = grpc_call_stack_element(call_stack, 0);
diff --git a/test/core/client_channel/lb_policies_test.c b/test/core/client_channel/lb_policies_test.c
index b9bd637d35..5255174852 100644
--- a/test/core/client_channel/lb_policies_test.c
+++ b/test/core/client_channel/lb_policies_test.c
@@ -63,8 +63,14 @@ typedef struct servers_fixture {
   grpc_metadata_array *request_metadata_recv;
 } servers_fixture;
 
+typedef struct request_sequences {
+  size_t n;
+  int *connections;
+  int *connectivity_states;
+} request_sequences;
+
 typedef void (*verifier_fn)(const servers_fixture *, grpc_channel *,
-                            const int *, const size_t);
+                            const request_sequences *, const size_t);
 
 typedef struct test_spec {
   size_t num_iters;
@@ -228,9 +234,24 @@ static void teardown_servers(servers_fixture *f) {
   gpr_free(f);
 }
 
+static request_sequences request_sequences_create(size_t n) {
+  request_sequences res;
+  res.n = n;
+  res.connections = gpr_malloc(sizeof(*res.connections) * n);
+  res.connectivity_states = gpr_malloc(sizeof(*res.connectivity_states) * n);
+  return res;
+}
+
+static void request_sequences_destroy(const request_sequences *rseqs) {
+  gpr_free(rseqs->connections);
+  gpr_free(rseqs->connectivity_states);
+}
+
 /** Returns connection sequence (server indices), which must be freed */
-static int *perform_request(servers_fixture *f, grpc_channel *client,
-                            request_data *rdata, const test_spec *spec) {
+static request_sequences perform_request(servers_fixture *f,
+                                         grpc_channel *client,
+                                         request_data *rdata,
+                                         const test_spec *spec) {
   grpc_call *c;
   int s_idx;
   int *s_valid;
@@ -240,11 +261,10 @@ static int *perform_request(servers_fixture *f, grpc_channel *client,
   size_t i, iter_num;
   grpc_event ev;
   int read_tag;
-  int *connection_sequence;
   int completed_client;
+  const request_sequences sequences = request_sequences_create(spec->num_iters);
 
   s_valid = gpr_malloc(sizeof(int) * f->num_servers);
-  connection_sequence = gpr_malloc(sizeof(int) * spec->num_iters);
 
   for (iter_num = 0; iter_num < spec->num_iters; iter_num++) {
     cq_verifier *cqv = cq_verifier_create(f->cq);
@@ -261,7 +281,7 @@ static int *perform_request(servers_fixture *f, grpc_channel *client,
       }
     }
 
-    connection_sequence[iter_num] = -1;
+    sequences.connections[iter_num] = -1;
     grpc_metadata_array_init(&rdata->initial_metadata_recv);
     grpc_metadata_array_init(&rdata->trailing_metadata_recv);
 
@@ -306,12 +326,14 @@ static int *perform_request(servers_fixture *f, grpc_channel *client,
                grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL));
 
     s_idx = -1;
-    while (
-        (ev = grpc_completion_queue_next(
-             f->cq, GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10 * RETRY_TIMEOUT), NULL))
-            .type != GRPC_QUEUE_TIMEOUT) {
+    while ((ev = grpc_completion_queue_next(
+                f->cq, GRPC_TIMEOUT_MILLIS_TO_DEADLINE(RETRY_TIMEOUT), NULL))
+               .type != GRPC_QUEUE_TIMEOUT) {
       GPR_ASSERT(ev.type == GRPC_OP_COMPLETE);
       read_tag = ((int)(intptr_t)ev.tag);
+      const grpc_connectivity_state conn_state =
+          grpc_channel_check_connectivity_state(client, 0);
+      sequences.connectivity_states[iter_num] = conn_state;
       gpr_log(GPR_DEBUG, "EVENT: success:%d, type:%d, tag:%d iter:%" PRIuPTR,
               ev.success, ev.type, read_tag, iter_num);
       if (ev.success && read_tag >= 1000) {
@@ -319,7 +341,7 @@ static int *perform_request(servers_fixture *f, grpc_channel *client,
         /* only server notifications for non-shutdown events */
         s_idx = read_tag - 1000;
         s_valid[s_idx] = 1;
-        connection_sequence[iter_num] = s_idx;
+        sequences.connections[iter_num] = s_idx;
         break;
       } else if (read_tag == 1) {
         gpr_log(GPR_DEBUG, "client timed out");
@@ -382,10 +404,9 @@ static int *perform_request(servers_fixture *f, grpc_channel *client,
       }
     }
 
-    GPR_ASSERT(
-        grpc_completion_queue_next(
-            f->cq, GRPC_TIMEOUT_MILLIS_TO_DEADLINE(2 * RETRY_TIMEOUT), NULL)
-            .type == GRPC_QUEUE_TIMEOUT);
+    GPR_ASSERT(grpc_completion_queue_next(
+                   f->cq, GRPC_TIMEOUT_MILLIS_TO_DEADLINE(RETRY_TIMEOUT), NULL)
+                   .type == GRPC_QUEUE_TIMEOUT);
 
     grpc_metadata_array_destroy(&rdata->initial_metadata_recv);
     grpc_metadata_array_destroy(&rdata->trailing_metadata_recv);
@@ -402,7 +423,7 @@ static int *perform_request(servers_fixture *f, grpc_channel *client,
 
   gpr_free(s_valid);
 
-  return connection_sequence;
+  return sequences;
 }
 
 static grpc_call **perform_multirequest(servers_fixture *f,
@@ -442,62 +463,10 @@ static grpc_call **perform_multirequest(servers_fixture *f,
   return calls;
 }
 
-static void assert_channel_connectivity(grpc_channel *ch,
-                                        size_t num_accepted_conn_states,
-                                        int accepted_conn_state, ...) {
-  size_t i;
-  grpc_channel_stack *client_stack;
-  grpc_channel_element *client_channel_filter;
-  grpc_connectivity_state actual_conn_state;
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  va_list ap;
-
-  client_stack = grpc_channel_get_channel_stack(ch);
-  client_channel_filter = grpc_channel_stack_last_element(client_stack);
-
-  actual_conn_state = grpc_client_channel_check_connectivity_state(
-      &exec_ctx, client_channel_filter, 0 /* don't try to connect */);
-  grpc_exec_ctx_finish(&exec_ctx);
-  va_start(ap, accepted_conn_state);
-  for (i = 0; i < num_accepted_conn_states; i++) {
-    if ((int)actual_conn_state == accepted_conn_state) {
-      break;
-    }
-    accepted_conn_state = va_arg(ap, grpc_connectivity_state);
-  }
-  va_end(ap);
-  if (i == num_accepted_conn_states) {
-    char **accepted_strs =
-        gpr_malloc(sizeof(char *) * num_accepted_conn_states);
-    char *accepted_str_joined;
-    va_start(ap, accepted_conn_state);
-    for (i = 0; i < num_accepted_conn_states; i++) {
-      GPR_ASSERT(gpr_asprintf(&accepted_strs[i], "%d", accepted_conn_state) >
-                 0);
-      accepted_conn_state = va_arg(ap, grpc_connectivity_state);
-    }
-    va_end(ap);
-    accepted_str_joined = gpr_strjoin_sep((const char **)accepted_strs,
-                                          num_accepted_conn_states, ", ", NULL);
-    gpr_log(
-        GPR_ERROR,
-        "Channel connectivity assertion failed: expected <one of [%s]>, got %d",
-        accepted_str_joined, actual_conn_state);
-
-    for (i = 0; i < num_accepted_conn_states; i++) {
-      gpr_free(accepted_strs[i]);
-    }
-    gpr_free(accepted_strs);
-    gpr_free(accepted_str_joined);
-    abort();
-  }
-}
-
 void run_spec(const test_spec *spec) {
   grpc_channel *client;
   char *client_hostport;
   char *servers_hostports_str;
-  int *actual_connection_sequence;
   request_data rdata;
   servers_fixture *f;
   grpc_channel_args args;
@@ -525,14 +494,14 @@ void run_spec(const test_spec *spec) {
   gpr_log(GPR_INFO, "Testing '%s' with servers=%s client=%s", spec->description,
           servers_hostports_str, client_hostport);
 
-  actual_connection_sequence = perform_request(f, client, &rdata, spec);
+  const request_sequences sequences = perform_request(f, client, &rdata, spec);
 
-  spec->verifier(f, client, actual_connection_sequence, spec->num_iters);
+  spec->verifier(f, client, &sequences, spec->num_iters);
 
   gpr_free(client_hostport);
   gpr_free(servers_hostports_str);
-  gpr_free(actual_connection_sequence);
   gpr_free(rdata.call_details);
+  request_sequences_destroy(&sequences);
 
   grpc_channel_destroy(client); /* calls the LB's shutdown func */
   teardown_servers(f);
@@ -644,7 +613,7 @@ static void test_pending_calls(size_t concurrent_calls) {
 
 static void test_get_channel_info() {
   grpc_channel *channel =
-      grpc_insecure_channel_create("ipv4:127.0.0.1:1234", NULL, NULL);
+      grpc_insecure_channel_create("ipv4:127.0.0.1:1234", &args, NULL);
   // Ensures that resolver returns.
   grpc_channel_check_connectivity_state(channel, true /* try_to_connect */);
   // First, request no fields.  This is a no-op.
@@ -699,29 +668,43 @@ static void print_failed_expectations(const int *expected_connection_sequence,
 
 static void verify_vanilla_round_robin(const servers_fixture *f,
                                        grpc_channel *client,
-                                       const int *actual_connection_sequence,
+                                       const request_sequences *sequences,
                                        const size_t num_iters) {
-  int *expected_connection_sequence;
-  size_t i;
   const size_t expected_seq_length = f->num_servers;
 
   /* verify conn. seq. expectation */
   /* get the first sequence of "num_servers" elements */
-  expected_connection_sequence = gpr_malloc(sizeof(int) * expected_seq_length);
-  memcpy(expected_connection_sequence, actual_connection_sequence,
+  int *expected_connection_sequence =
+      gpr_malloc(sizeof(int) * expected_seq_length);
+  memcpy(expected_connection_sequence, sequences->connections,
          sizeof(int) * expected_seq_length);
 
-  for (i = 0; i < num_iters; i++) {
-    const int actual = actual_connection_sequence[i];
+  for (size_t i = 0; i < num_iters; i++) {
+    const int actual = sequences->connections[i];
     const int expected = expected_connection_sequence[i % expected_seq_length];
     if (actual != expected) {
-      print_failed_expectations(expected_connection_sequence,
-                                actual_connection_sequence, expected_seq_length,
-                                num_iters);
+      gpr_log(
+          GPR_ERROR,
+          "CONNECTION SEQUENCE FAILURE: expected %d, got %d at iteration #%d",
+          expected, actual, (int)i);
+      abort();
+    }
+  }
+
+  /* All servers are available, therefore all client subchannels are READY, even
+   * when we only need one for the client channel state to be READY */
+  for (size_t i = 0; i < sequences->n; i++) {
+    const grpc_connectivity_state actual = sequences->connectivity_states[i];
+    const grpc_connectivity_state expected = GRPC_CHANNEL_READY;
+    if (actual != expected) {
+      gpr_log(GPR_ERROR,
+              "CONNECTIVITY STATUS SEQUENCE FAILURE: expected '%s', got '%s' "
+              "at iteration #%d",
+              grpc_connectivity_state_name(expected),
+              grpc_connectivity_state_name(actual), (int)i);
       abort();
     }
   }
-  assert_channel_connectivity(client, 1, GRPC_CHANNEL_READY);
 
   gpr_free(expected_connection_sequence);
 }
@@ -730,7 +713,7 @@ static void verify_vanilla_round_robin(const servers_fixture *f,
  * given in "f") are killed */
 static void verify_vanishing_floor_round_robin(
     const servers_fixture *f, grpc_channel *client,
-    const int *actual_connection_sequence, const size_t num_iters) {
+    const request_sequences *sequences, const size_t num_iters) {
   int *expected_connection_sequence;
   const size_t expected_seq_length = 2;
   size_t i;
@@ -738,57 +721,83 @@ static void verify_vanishing_floor_round_robin(
   /* verify conn. seq. expectation */
   /* copy the first full sequence (without -1s) */
   expected_connection_sequence = gpr_malloc(sizeof(int) * expected_seq_length);
-  memcpy(expected_connection_sequence, actual_connection_sequence + 2,
+  memcpy(expected_connection_sequence, sequences->connections + 2,
          expected_seq_length * sizeof(int));
 
   /* first two elements of the sequence should be [0 (1st server), -1 (failure)]
    */
-  GPR_ASSERT(actual_connection_sequence[0] == 0);
-  GPR_ASSERT(actual_connection_sequence[1] == -1);
+  GPR_ASSERT(sequences->connections[0] == 0);
+  GPR_ASSERT(sequences->connections[1] == -1);
 
   /* the next two element must be [3, 0], repeating from that point: the 3 is
    * brought forth by servers 1 and 2 disappearing after the intial pick of 0 */
-  GPR_ASSERT(actual_connection_sequence[2] == 3);
-  GPR_ASSERT(actual_connection_sequence[3] == 0);
+  GPR_ASSERT(sequences->connections[2] == 3);
+  GPR_ASSERT(sequences->connections[3] == 0);
 
   /* make sure that the expectation obliges */
   for (i = 2; i < num_iters; i++) {
-    const int actual = actual_connection_sequence[i];
+    const int actual = sequences->connections[i];
     const int expected = expected_connection_sequence[i % expected_seq_length];
     if (actual != expected) {
       print_failed_expectations(expected_connection_sequence,
-                                actual_connection_sequence, expected_seq_length,
+                                sequences->connections, expected_seq_length,
                                 num_iters);
       abort();
     }
   }
+
+  /* There's always at least one subchannel READY (connected), therefore the
+   * overall state of the client channel is READY at all times. */
+  for (i = 0; i < sequences->n; i++) {
+    const grpc_connectivity_state actual = sequences->connectivity_states[i];
+    const grpc_connectivity_state expected = GRPC_CHANNEL_READY;
+    if (actual != expected) {
+      gpr_log(GPR_ERROR,
+              "CONNECTIVITY STATUS SEQUENCE FAILURE: expected '%s', got '%s' "
+              "at iteration #%d",
+              grpc_connectivity_state_name(expected),
+              grpc_connectivity_state_name(actual), (int)i);
+      abort();
+    }
+  }
+
   gpr_free(expected_connection_sequence);
 }
 
-static void verify_total_carnage_round_robin(
-    const servers_fixture *f, grpc_channel *client,
-    const int *actual_connection_sequence, const size_t num_iters) {
-  size_t i;
-
-  for (i = 0; i < num_iters; i++) {
-    const int actual = actual_connection_sequence[i];
+static void verify_total_carnage_round_robin(const servers_fixture *f,
+                                             grpc_channel *client,
+                                             const request_sequences *sequences,
+                                             const size_t num_iters) {
+  for (size_t i = 0; i < num_iters; i++) {
+    const int actual = sequences->connections[i];
     const int expected = -1;
     if (actual != expected) {
-      gpr_log(GPR_ERROR, "FAILURE: expected %d, actual %d at iter %" PRIuPTR,
-              expected, actual, i);
+      gpr_log(
+          GPR_ERROR,
+          "CONNECTION SEQUENCE FAILURE: expected %d, got %d at iteration #%d",
+          expected, actual, (int)i);
       abort();
     }
   }
 
-  /* even though we know all the servers are dead, the client is still trying
-   * retrying, believing it's in a transient failure situation */
-  assert_channel_connectivity(client, 2, GRPC_CHANNEL_TRANSIENT_FAILURE,
-                              GRPC_CHANNEL_CONNECTING);
+  /* no server is ever available. The persistent state is TRANSIENT_FAILURE */
+  for (size_t i = 0; i < sequences->n; i++) {
+    const grpc_connectivity_state actual = sequences->connectivity_states[i];
+    const grpc_connectivity_state expected = GRPC_CHANNEL_TRANSIENT_FAILURE;
+    if (actual != expected) {
+      gpr_log(GPR_ERROR,
+              "CONNECTIVITY STATUS SEQUENCE FAILURE: expected '%s', got '%s' "
+              "at iteration #%d",
+              grpc_connectivity_state_name(expected),
+              grpc_connectivity_state_name(actual), (int)i);
+      abort();
+    }
+  }
 }
 
 static void verify_partial_carnage_round_robin(
     const servers_fixture *f, grpc_channel *client,
-    const int *actual_connection_sequence, const size_t num_iters) {
+    const request_sequences *sequences, const size_t num_iters) {
   int *expected_connection_sequence;
   size_t i;
   const size_t expected_seq_length = f->num_servers;
@@ -796,15 +805,15 @@ static void verify_partial_carnage_round_robin(
   /* verify conn. seq. expectation */
   /* get the first sequence of "num_servers" elements */
   expected_connection_sequence = gpr_malloc(sizeof(int) * expected_seq_length);
-  memcpy(expected_connection_sequence, actual_connection_sequence,
+  memcpy(expected_connection_sequence, sequences->connections,
          sizeof(int) * expected_seq_length);
 
   for (i = 0; i < num_iters / 2; i++) {
-    const int actual = actual_connection_sequence[i];
+    const int actual = sequences->connections[i];
     const int expected = expected_connection_sequence[i % expected_seq_length];
     if (actual != expected) {
       print_failed_expectations(expected_connection_sequence,
-                                actual_connection_sequence, expected_seq_length,
+                                sequences->connections, expected_seq_length,
                                 num_iters);
       abort();
     }
@@ -812,13 +821,34 @@ static void verify_partial_carnage_round_robin(
 
   /* second half of the iterations go without response */
   for (; i < num_iters; i++) {
-    GPR_ASSERT(actual_connection_sequence[i] == -1);
+    GPR_ASSERT(sequences->connections[i] == -1);
+  }
+
+  /* We can assert that the first client channel state should be READY, when all
+   * servers were available; and that the last one should be TRANSIENT_FAILURE,
+   * after all servers are gone. */
+  grpc_connectivity_state actual = sequences->connectivity_states[0];
+  grpc_connectivity_state expected = GRPC_CHANNEL_READY;
+  if (actual != expected) {
+    gpr_log(GPR_ERROR,
+            "CONNECTIVITY STATUS SEQUENCE FAILURE: expected '%s', got '%s' "
+            "at iteration #%d",
+            grpc_connectivity_state_name(expected),
+            grpc_connectivity_state_name(actual), 0);
+    abort();
+  }
+
+  actual = sequences->connectivity_states[num_iters - 1];
+  expected = GRPC_CHANNEL_TRANSIENT_FAILURE;
+  if (actual != expected) {
+    gpr_log(GPR_ERROR,
+            "CONNECTIVITY STATUS SEQUENCE FAILURE: expected '%s', got '%s' "
+            "at iteration #%d",
+            grpc_connectivity_state_name(expected),
+            grpc_connectivity_state_name(actual), (int)num_iters - 1);
+    abort();
   }
 
-  /* even though we know all the servers are dead, the client is still trying
-   * retrying, believing it's in a transient failure situation */
-  assert_channel_connectivity(client, 2, GRPC_CHANNEL_TRANSIENT_FAILURE,
-                              GRPC_CHANNEL_CONNECTING);
   gpr_free(expected_connection_sequence);
 }
 
@@ -841,15 +871,14 @@ static void dump_array(const char *desc, const int *data, const size_t count) {
 
 static void verify_rebirth_round_robin(const servers_fixture *f,
                                        grpc_channel *client,
-                                       const int *actual_connection_sequence,
+                                       const request_sequences *sequences,
                                        const size_t num_iters) {
   int *expected_connection_sequence;
   size_t i, j, unique_seq_last_idx, unique_seq_first_idx;
   const size_t expected_seq_length = f->num_servers;
   int *seen_elements;
 
-  dump_array("actual_connection_sequence", actual_connection_sequence,
-             num_iters);
+  dump_array("actual_connection_sequence", sequences->connections, num_iters);
 
   /* verify conn. seq. expectation */
   /* get the first unique run of length "num_servers". */
@@ -860,13 +889,13 @@ static void verify_rebirth_round_robin(const servers_fixture *f,
 
   memset(seen_elements, 0, sizeof(int) * expected_seq_length);
   for (i = 0; i < num_iters; i++) {
-    if (actual_connection_sequence[i] < 0 ||
-        seen_elements[actual_connection_sequence[i]] != 0) {
+    if (sequences->connections[i] < 0 ||
+        seen_elements[sequences->connections[i]] != 0) {
       /* if anything breaks the uniqueness of the run, back to square zero */
       memset(seen_elements, 0, sizeof(int) * expected_seq_length);
       continue;
     }
-    seen_elements[actual_connection_sequence[i]] = 1;
+    seen_elements[sequences->connections[i]] = 1;
     for (j = 0; j < expected_seq_length; j++) {
       if (seen_elements[j] == 0) break;
     }
@@ -885,30 +914,72 @@ static void verify_rebirth_round_robin(const servers_fixture *f,
 
   unique_seq_first_idx = (unique_seq_last_idx - expected_seq_length + 1);
   memcpy(expected_connection_sequence,
-         actual_connection_sequence + unique_seq_first_idx,
+         sequences->connections + unique_seq_first_idx,
          sizeof(int) * expected_seq_length);
 
   /* first iteration succeeds */
-  GPR_ASSERT(actual_connection_sequence[0] != -1);
+  GPR_ASSERT(sequences->connections[0] != -1);
   /* then we fail for a while... */
-  GPR_ASSERT(actual_connection_sequence[1] == -1);
+  GPR_ASSERT(sequences->connections[1] == -1);
   /* ... but should be up at "unique_seq_first_idx" */
-  GPR_ASSERT(actual_connection_sequence[unique_seq_first_idx] != -1);
+  GPR_ASSERT(sequences->connections[unique_seq_first_idx] != -1);
 
   for (j = 0, i = unique_seq_first_idx; i < num_iters; i++) {
-    const int actual = actual_connection_sequence[i];
+    const int actual = sequences->connections[i];
     const int expected =
         expected_connection_sequence[j++ % expected_seq_length];
     if (actual != expected) {
       print_failed_expectations(expected_connection_sequence,
-                                actual_connection_sequence, expected_seq_length,
+                                sequences->connections, expected_seq_length,
                                 num_iters);
       abort();
     }
   }
 
-  /* things are fine once the servers are brought back up */
-  assert_channel_connectivity(client, 1, GRPC_CHANNEL_READY);
+  /* We can assert that the first client channel state should be READY, when all
+   * servers were available; same thing for the last one. In the middle
+   * somewhere there must exist at least one TRANSIENT_FAILURE */
+  grpc_connectivity_state actual = sequences->connectivity_states[0];
+  grpc_connectivity_state expected = GRPC_CHANNEL_READY;
+  if (actual != expected) {
+    gpr_log(GPR_ERROR,
+            "CONNECTIVITY STATUS SEQUENCE FAILURE: expected '%s', got '%s' "
+            "at iteration #%d",
+            grpc_connectivity_state_name(expected),
+            grpc_connectivity_state_name(actual), 0);
+    abort();
+  }
+
+  actual = sequences->connectivity_states[num_iters - 1];
+  expected = GRPC_CHANNEL_READY;
+  if (actual != expected) {
+    gpr_log(GPR_ERROR,
+            "CONNECTIVITY STATUS SEQUENCE FAILURE: expected '%s', got '%s' "
+            "at iteration #%d",
+            grpc_connectivity_state_name(expected),
+            grpc_connectivity_state_name(actual), (int)num_iters - 1);
+    abort();
+  }
+
+  bool found_failure_status = false;
+  for (i = 1; i < sequences->n - 1; i++) {
+    if (sequences->connectivity_states[i] == GRPC_CHANNEL_TRANSIENT_FAILURE) {
+      found_failure_status = true;
+      break;
+    }
+  }
+  if (!found_failure_status) {
+    gpr_log(
+        GPR_ERROR,
+        "CONNECTIVITY STATUS SEQUENCE FAILURE: "
+        "GRPC_CHANNEL_TRANSIENT_FAILURE status not found. Got the following "
+        "instead:");
+    for (i = 0; i < num_iters; i++) {
+      gpr_log(GPR_ERROR, "[%d]: %s", (int)i,
+              grpc_connectivity_state_name(sequences->connectivity_states[i]));
+    }
+  }
+
   gpr_free(expected_connection_sequence);
   gpr_free(seen_elements);
 }
@@ -949,7 +1020,7 @@ int main(int argc, char **argv) {
    * This should knock down the server bound to be selected next */
   test_spec_reset(spec);
   spec->verifier = verify_vanishing_floor_round_robin;
-  spec->description = "test_kill_all_server_at_2nd_iteration";
+  spec->description = "test_kill_middle_servers_at_2nd_iteration";
   for (i = 1; i < NUM_SERVERS - 1; i++) {
     spec->kill_at[1][i] = 1;
   }
diff --git a/test/core/end2end/end2end_nosec_tests.c b/test/core/end2end/end2end_nosec_tests.c
index 457c188ce5..663489082f 100644
--- a/test/core/end2end/end2end_nosec_tests.c
+++ b/test/core/end2end/end2end_nosec_tests.c
@@ -75,6 +75,8 @@ extern void filter_call_init_fails(grpc_end2end_test_config config);
 extern void filter_call_init_fails_pre_init(void);
 extern void filter_causes_close(grpc_end2end_test_config config);
 extern void filter_causes_close_pre_init(void);
+extern void filter_latency(grpc_end2end_test_config config);
+extern void filter_latency_pre_init(void);
 extern void graceful_server_shutdown(grpc_end2end_test_config config);
 extern void graceful_server_shutdown_pre_init(void);
 extern void high_initial_seqno(grpc_end2end_test_config config);
@@ -153,6 +155,7 @@ void grpc_end2end_tests_pre_init(void) {
   empty_batch_pre_init();
   filter_call_init_fails_pre_init();
   filter_causes_close_pre_init();
+  filter_latency_pre_init();
   graceful_server_shutdown_pre_init();
   high_initial_seqno_pre_init();
   hpack_size_pre_init();
@@ -207,6 +210,7 @@ void grpc_end2end_tests(int argc, char **argv,
     empty_batch(config);
     filter_call_init_fails(config);
     filter_causes_close(config);
+    filter_latency(config);
     graceful_server_shutdown(config);
     high_initial_seqno(config);
     hpack_size(config);
@@ -304,6 +308,10 @@ void grpc_end2end_tests(int argc, char **argv,
       filter_causes_close(config);
       continue;
     }
+    if (0 == strcmp("filter_latency", argv[i])) {
+      filter_latency(config);
+      continue;
+    }
     if (0 == strcmp("graceful_server_shutdown", argv[i])) {
       graceful_server_shutdown(config);
       continue;
diff --git a/test/core/end2end/end2end_tests.c b/test/core/end2end/end2end_tests.c
index dc2d68a486..25c7c62fde 100644
--- a/test/core/end2end/end2end_tests.c
+++ b/test/core/end2end/end2end_tests.c
@@ -77,6 +77,8 @@ extern void filter_call_init_fails(grpc_end2end_test_config config);
 extern void filter_call_init_fails_pre_init(void);
 extern void filter_causes_close(grpc_end2end_test_config config);
 extern void filter_causes_close_pre_init(void);
+extern void filter_latency(grpc_end2end_test_config config);
+extern void filter_latency_pre_init(void);
 extern void graceful_server_shutdown(grpc_end2end_test_config config);
 extern void graceful_server_shutdown_pre_init(void);
 extern void high_initial_seqno(grpc_end2end_test_config config);
@@ -156,6 +158,7 @@ void grpc_end2end_tests_pre_init(void) {
   empty_batch_pre_init();
   filter_call_init_fails_pre_init();
   filter_causes_close_pre_init();
+  filter_latency_pre_init();
   graceful_server_shutdown_pre_init();
   high_initial_seqno_pre_init();
   hpack_size_pre_init();
@@ -211,6 +214,7 @@ void grpc_end2end_tests(int argc, char **argv,
     empty_batch(config);
     filter_call_init_fails(config);
     filter_causes_close(config);
+    filter_latency(config);
     graceful_server_shutdown(config);
     high_initial_seqno(config);
     hpack_size(config);
@@ -312,6 +316,10 @@ void grpc_end2end_tests(int argc, char **argv,
       filter_causes_close(config);
       continue;
     }
+    if (0 == strcmp("filter_latency", argv[i])) {
+      filter_latency(config);
+      continue;
+    }
     if (0 == strcmp("graceful_server_shutdown", argv[i])) {
       graceful_server_shutdown(config);
       continue;
diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py
index 5f54b821a7..201a92a1fd 100755
--- a/test/core/end2end/gen_build_yaml.py
+++ b/test/core/end2end/gen_build_yaml.py
@@ -111,6 +111,7 @@ END2END_TESTS = {
     'empty_batch': default_test_options,
     'filter_causes_close': default_test_options,
     'filter_call_init_fails': default_test_options,
+    'filter_latency': default_test_options,
     'graceful_server_shutdown': default_test_options._replace(cpu_cost=LOWCPU),
     'hpack_size': default_test_options._replace(proxyable=False,
                                                 traceable=False),
diff --git a/test/core/end2end/tests/filter_latency.c b/test/core/end2end/tests/filter_latency.c
new file mode 100644
index 0000000000..37ce3b1222
--- /dev/null
+++ b/test/core/end2end/tests/filter_latency.c
@@ -0,0 +1,359 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "test/core/end2end/end2end_tests.h"
+
+#include <limits.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <grpc/byte_buffer.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/time.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/lib/channel/channel_stack_builder.h"
+#include "src/core/lib/surface/channel_init.h"
+#include "test/core/end2end/cq_verifier.h"
+
+enum { TIMEOUT = 200000 };
+
+static bool g_enable_filter = false;
+static gpr_mu g_mu;
+static gpr_timespec g_client_latency;
+static gpr_timespec g_server_latency;
+
+static void *tag(intptr_t t) { return (void *)t; }
+
+static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config,
+                                            const char *test_name,
+                                            grpc_channel_args *client_args,
+                                            grpc_channel_args *server_args) {
+  grpc_end2end_test_fixture f;
+  gpr_log(GPR_INFO, "%s/%s", test_name, config.name);
+  f = config.create_fixture(client_args, server_args);
+  config.init_server(&f, server_args);
+  config.init_client(&f, client_args);
+  return f;
+}
+
+static gpr_timespec n_seconds_time(int n) {
+  return GRPC_TIMEOUT_SECONDS_TO_DEADLINE(n);
+}
+
+static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); }
+
+static void drain_cq(grpc_completion_queue *cq) {
+  grpc_event ev;
+  do {
+    ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL);
+  } while (ev.type != GRPC_QUEUE_SHUTDOWN);
+}
+
+static void shutdown_server(grpc_end2end_test_fixture *f) {
+  if (!f->server) return;
+  grpc_server_shutdown_and_notify(f->server, f->cq, tag(1000));
+  GPR_ASSERT(grpc_completion_queue_pluck(
+                 f->cq, tag(1000), GRPC_TIMEOUT_SECONDS_TO_DEADLINE(5), NULL)
+                 .type == GRPC_OP_COMPLETE);
+  grpc_server_destroy(f->server);
+  f->server = NULL;
+}
+
+static void shutdown_client(grpc_end2end_test_fixture *f) {
+  if (!f->client) return;
+  grpc_channel_destroy(f->client);
+  f->client = NULL;
+}
+
+static void end_test(grpc_end2end_test_fixture *f) {
+  shutdown_server(f);
+  shutdown_client(f);
+
+  grpc_completion_queue_shutdown(f->cq);
+  drain_cq(f->cq);
+  grpc_completion_queue_destroy(f->cq);
+}
+
+// Simple request via a server filter that saves the reported latency value.
+static void test_request(grpc_end2end_test_config config) {
+  grpc_call *c;
+  grpc_call *s;
+  grpc_slice request_payload_slice =
+      grpc_slice_from_copied_string("hello world");
+  grpc_byte_buffer *request_payload =
+      grpc_raw_byte_buffer_create(&request_payload_slice, 1);
+  gpr_timespec deadline = five_seconds_time();
+  grpc_end2end_test_fixture f =
+      begin_test(config, "filter_latency", NULL, NULL);
+  cq_verifier *cqv = cq_verifier_create(f.cq);
+  grpc_op ops[6];
+  grpc_op *op;
+  grpc_metadata_array initial_metadata_recv;
+  grpc_metadata_array trailing_metadata_recv;
+  grpc_metadata_array request_metadata_recv;
+  grpc_byte_buffer *request_payload_recv = NULL;
+  grpc_call_details call_details;
+  grpc_status_code status;
+  grpc_call_error error;
+  char *details = NULL;
+  size_t details_capacity = 0;
+  int was_cancelled = 2;
+
+  gpr_mu_lock(&g_mu);
+  g_client_latency = gpr_time_0(GPR_TIMESPAN);
+  g_server_latency = gpr_time_0(GPR_TIMESPAN);
+  gpr_mu_unlock(&g_mu);
+  const gpr_timespec start_time = gpr_now(GPR_CLOCK_MONOTONIC);
+
+  c = grpc_channel_create_call(f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq,
+                               "/foo", "foo.test.google.fr", deadline, NULL);
+  GPR_ASSERT(c);
+
+  grpc_metadata_array_init(&initial_metadata_recv);
+  grpc_metadata_array_init(&trailing_metadata_recv);
+  grpc_metadata_array_init(&request_metadata_recv);
+  grpc_call_details_init(&call_details);
+
+  memset(ops, 0, sizeof(ops));
+  op = ops;
+  op->op = GRPC_OP_SEND_INITIAL_METADATA;
+  op->data.send_initial_metadata.count = 0;
+  op->data.send_initial_metadata.metadata = NULL;
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+  op->op = GRPC_OP_SEND_MESSAGE;
+  op->data.send_message = request_payload;
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+  op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+  op->op = GRPC_OP_RECV_INITIAL_METADATA;
+  op->data.recv_initial_metadata = &initial_metadata_recv;
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+  op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
+  op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
+  op->data.recv_status_on_client.status = &status;
+  op->data.recv_status_on_client.status_details = &details;
+  op->data.recv_status_on_client.status_details_capacity = &details_capacity;
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+  error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL);
+  GPR_ASSERT(GRPC_CALL_OK == error);
+
+  error =
+      grpc_server_request_call(f.server, &s, &call_details,
+                               &request_metadata_recv, f.cq, f.cq, tag(101));
+  GPR_ASSERT(GRPC_CALL_OK == error);
+
+  CQ_EXPECT_COMPLETION(cqv, tag(101), 1);
+  cq_verify(cqv);
+
+  memset(ops, 0, sizeof(ops));
+  op = ops;
+  op->op = GRPC_OP_SEND_INITIAL_METADATA;
+  op->data.send_initial_metadata.count = 0;
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+  op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
+  op->data.send_status_from_server.trailing_metadata_count = 0;
+  op->data.send_status_from_server.status = GRPC_STATUS_UNIMPLEMENTED;
+  op->data.send_status_from_server.status_details = "xyz";
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+  op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
+  op->data.recv_close_on_server.cancelled = &was_cancelled;
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+  error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(102), NULL);
+  GPR_ASSERT(GRPC_CALL_OK == error);
+
+  CQ_EXPECT_COMPLETION(cqv, tag(102), 1);
+  CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
+  cq_verify(cqv);
+
+  GPR_ASSERT(status == GRPC_STATUS_UNIMPLEMENTED);
+  GPR_ASSERT(0 == strcmp(details, "xyz"));
+
+  gpr_free(details);
+  grpc_metadata_array_destroy(&initial_metadata_recv);
+  grpc_metadata_array_destroy(&trailing_metadata_recv);
+  grpc_metadata_array_destroy(&request_metadata_recv);
+  grpc_call_details_destroy(&call_details);
+
+  grpc_call_destroy(s);
+  grpc_call_destroy(c);
+
+  const gpr_timespec end_time = gpr_now(GPR_CLOCK_MONOTONIC);
+  const gpr_timespec max_latency = gpr_time_sub(end_time, start_time);
+
+  gpr_mu_lock(&g_mu);
+  GPR_ASSERT(gpr_time_cmp(max_latency, g_client_latency) >= 0);
+  GPR_ASSERT(gpr_time_cmp(gpr_time_0(GPR_TIMESPAN), g_client_latency) < 0);
+  GPR_ASSERT(gpr_time_cmp(max_latency, g_server_latency) >= 0);
+  GPR_ASSERT(gpr_time_cmp(gpr_time_0(GPR_TIMESPAN), g_server_latency) < 0);
+  // Server latency should always be smaller than client latency.
+  GPR_ASSERT(gpr_time_cmp(g_server_latency, g_client_latency) < 0);
+  gpr_mu_unlock(&g_mu);
+
+  cq_verifier_destroy(cqv);
+
+  grpc_byte_buffer_destroy(request_payload);
+  grpc_byte_buffer_destroy(request_payload_recv);
+
+  end_test(&f);
+  config.tear_down_data(&f);
+}
+
+/*******************************************************************************
+ * Test latency filter
+ */
+
+static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
+                                  grpc_call_element *elem,
+                                  grpc_call_element_args *args) {
+  return GRPC_ERROR_NONE;
+}
+
+static void client_destroy_call_elem(grpc_exec_ctx *exec_ctx,
+                                     grpc_call_element *elem,
+                                     const grpc_call_final_info *final_info,
+                                     void *and_free_memory) {
+  gpr_mu_lock(&g_mu);
+  g_client_latency = final_info->stats.latency;
+  gpr_mu_unlock(&g_mu);
+}
+
+static void server_destroy_call_elem(grpc_exec_ctx *exec_ctx,
+                                     grpc_call_element *elem,
+                                     const grpc_call_final_info *final_info,
+                                     void *and_free_memory) {
+  gpr_mu_lock(&g_mu);
+  g_server_latency = final_info->stats.latency;
+  gpr_mu_unlock(&g_mu);
+}
+
+static void init_channel_elem(grpc_exec_ctx *exec_ctx,
+                              grpc_channel_element *elem,
+                              grpc_channel_element_args *args) {}
+
+static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
+                                 grpc_channel_element *elem) {}
+
+static const grpc_channel_filter test_client_filter = {
+    grpc_call_next_op,
+    grpc_channel_next_op,
+    0,
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
+    client_destroy_call_elem,
+    0,
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_call_next_get_peer,
+    grpc_channel_next_get_info,
+    "client_filter_latency"};
+
+static const grpc_channel_filter test_server_filter = {
+    grpc_call_next_op,
+    grpc_channel_next_op,
+    0,
+    init_call_elem,
+    grpc_call_stack_ignore_set_pollset_or_pollset_set,
+    server_destroy_call_elem,
+    0,
+    init_channel_elem,
+    destroy_channel_elem,
+    grpc_call_next_get_peer,
+    grpc_channel_next_get_info,
+    "server_filter_latency"};
+
+/*******************************************************************************
+ * Registration
+ */
+
+static bool maybe_add_filter(grpc_channel_stack_builder *builder, void *arg) {
+  grpc_channel_filter *filter = arg;
+  if (g_enable_filter) {
+    // Want to add the filter as close to the end as possible, to make
+    // sure that all of the filters work well together.  However, we
+    // can't add it at the very end, because the connected channel filter
+    // must be the last one.  So we add it right before the last one.
+    grpc_channel_stack_builder_iterator *it =
+        grpc_channel_stack_builder_create_iterator_at_last(builder);
+    GPR_ASSERT(grpc_channel_stack_builder_move_prev(it));
+    const bool retval =
+        grpc_channel_stack_builder_add_filter_before(it, filter, NULL, NULL);
+    grpc_channel_stack_builder_iterator_destroy(it);
+    return retval;
+  } else {
+    return true;
+  }
+}
+
+static void init_plugin(void) {
+  gpr_mu_init(&g_mu);
+  grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX,
+                                   maybe_add_filter,
+                                   (void *)&test_client_filter);
+  grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX,
+                                   maybe_add_filter,
+                                   (void *)&test_client_filter);
+  grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX,
+                                   maybe_add_filter,
+                                   (void *)&test_server_filter);
+}
+
+static void destroy_plugin(void) { gpr_mu_destroy(&g_mu); }
+
+void filter_latency(grpc_end2end_test_config config) {
+  g_enable_filter = true;
+  test_request(config);
+  g_enable_filter = false;
+}
+
+void filter_latency_pre_init(void) {
+  grpc_register_plugin(init_plugin, destroy_plugin);
+}
diff --git a/test/core/internal_api_canaries/iomgr.c b/test/core/internal_api_canaries/iomgr.c
index f1efa87a69..de03c47c13 100644
--- a/test/core/internal_api_canaries/iomgr.c
+++ b/test/core/internal_api_canaries/iomgr.c
@@ -85,7 +85,8 @@ static void test_code(void) {
                                  grpc_endpoint_shutdown,
                                  grpc_endpoint_destroy,
                                  grpc_endpoint_get_resource_user,
-                                 grpc_endpoint_get_peer};
+                                 grpc_endpoint_get_peer,
+                                 grpc_endpoint_get_fd};
   endpoint.vtable = &vtable;
 
   grpc_endpoint_read(&exec_ctx, &endpoint, NULL, NULL);
diff --git a/test/core/iomgr/socket_utils_test.c b/test/core/iomgr/socket_utils_test.c
index 67bc914c15..7eee2d1d10 100644
--- a/test/core/iomgr/socket_utils_test.c
+++ b/test/core/iomgr/socket_utils_test.c
@@ -39,13 +39,57 @@
 #include "src/core/lib/iomgr/socket_utils_posix.h"
 
 #include <errno.h>
+#include <netinet/ip.h>
 #include <string.h>
 
+#include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/useful.h>
+#include "src/core/lib/iomgr/socket_mutator.h"
 #include "test/core/util/test_config.h"
 
+struct test_socket_mutator {
+  grpc_socket_mutator base;
+  int option_value;
+};
+
+static bool mutate_fd(int fd, grpc_socket_mutator *mutator) {
+  int newval;
+  socklen_t intlen = sizeof(newval);
+  struct test_socket_mutator *m = (struct test_socket_mutator *)mutator;
+
+  if (0 != setsockopt(fd, IPPROTO_IP, IP_TOS, &m->option_value,
+                      sizeof(m->option_value))) {
+    return false;
+  }
+  if (0 != getsockopt(fd, IPPROTO_IP, IP_TOS, &newval, &intlen)) {
+    return false;
+  }
+  if (newval != m->option_value) {
+    return false;
+  }
+  return true;
+}
+
+static void destroy_test_mutator(grpc_socket_mutator *mutator) {
+  struct test_socket_mutator *m = (struct test_socket_mutator *)mutator;
+  gpr_free(m);
+}
+
+static int compare_test_mutator(grpc_socket_mutator *a,
+                                grpc_socket_mutator *b) {
+  struct test_socket_mutator *ma = (struct test_socket_mutator *)a;
+  struct test_socket_mutator *mb = (struct test_socket_mutator *)b;
+  return GPR_ICMP(ma->option_value, mb->option_value);
+}
+
+static const grpc_socket_mutator_vtable mutator_vtable = {
+    mutate_fd, compare_test_mutator, destroy_test_mutator};
+
 int main(int argc, char **argv) {
   int sock;
+  grpc_error *err;
   grpc_test_init(argc, argv);
 
   sock = socket(PF_INET, SOCK_STREAM, 0);
@@ -68,6 +112,29 @@ int main(int argc, char **argv) {
   GPR_ASSERT(GRPC_LOG_IF_ERROR("set_socket_low_latency",
                                grpc_set_socket_low_latency(sock, 0)));
 
+  struct test_socket_mutator mutator;
+  grpc_socket_mutator_init(&mutator.base, &mutator_vtable);
+
+  mutator.option_value = IPTOS_LOWDELAY;
+  GPR_ASSERT(GRPC_LOG_IF_ERROR(
+      "set_socket_with_mutator",
+      grpc_set_socket_with_mutator(sock, (grpc_socket_mutator *)&mutator)));
+
+  mutator.option_value = IPTOS_THROUGHPUT;
+  GPR_ASSERT(GRPC_LOG_IF_ERROR(
+      "set_socket_with_mutator",
+      grpc_set_socket_with_mutator(sock, (grpc_socket_mutator *)&mutator)));
+
+  mutator.option_value = IPTOS_RELIABILITY;
+  GPR_ASSERT(GRPC_LOG_IF_ERROR(
+      "set_socket_with_mutator",
+      grpc_set_socket_with_mutator(sock, (grpc_socket_mutator *)&mutator)));
+
+  mutator.option_value = -1;
+  err = grpc_set_socket_with_mutator(sock, (grpc_socket_mutator *)&mutator);
+  GPR_ASSERT(err != GRPC_ERROR_NONE);
+  GRPC_ERROR_UNREF(err);
+
   close(sock);
 
   return 0;
diff --git a/test/core/network_benchmarks/low_level_ping_pong.c b/test/core/network_benchmarks/low_level_ping_pong.c
index 9038d07675..0b2a920873 100644
--- a/test/core/network_benchmarks/low_level_ping_pong.c
+++ b/test/core/network_benchmarks/low_level_ping_pong.c
@@ -56,6 +56,7 @@
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
+#include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/socket_utils_posix.h"
 
 typedef struct fd_pair {
@@ -229,12 +230,12 @@ static int blocking_write_bytes(struct thread_args *args, char *buf) {
    on the scenario we're using.
  */
 static int set_socket_nonblocking(thread_args *args) {
-  if (!grpc_set_socket_nonblocking(args->fds.read_fd, 1)) {
-    gpr_log(GPR_ERROR, "Unable to set socket nonblocking: %s", strerror(errno));
+  if (!GRPC_LOG_IF_ERROR("Unable to set read socket nonblocking",
+                         grpc_set_socket_nonblocking(args->fds.read_fd, 1))) {
     return -1;
   }
-  if (!grpc_set_socket_nonblocking(args->fds.write_fd, 1)) {
-    gpr_log(GPR_ERROR, "Unable to set socket nonblocking: %s", strerror(errno));
+  if (!GRPC_LOG_IF_ERROR("Unable to set write socket nonblocking",
+                         grpc_set_socket_nonblocking(args->fds.write_fd, 1))) {
     return -1;
   }
   return 0;
@@ -347,10 +348,16 @@ static int create_listening_socket(struct sockaddr *port, socklen_t len) {
     goto error;
   }
 
-  if (!grpc_set_socket_cloexec(fd, 1) || !grpc_set_socket_low_latency(fd, 1) ||
-      !grpc_set_socket_reuse_addr(fd, 1)) {
-    gpr_log(GPR_ERROR, "Unable to configure socket %d: %s", fd,
-            strerror(errno));
+  if (!GRPC_LOG_IF_ERROR("Failed to set listening socket cloexec",
+                         grpc_set_socket_cloexec(fd, 1))) {
+    goto error;
+  }
+  if (!GRPC_LOG_IF_ERROR("Failed to set listening socket low latency",
+                         grpc_set_socket_low_latency(fd, 1))) {
+    goto error;
+  }
+  if (!GRPC_LOG_IF_ERROR("Failed to set listening socket reuse addr",
+                         grpc_set_socket_reuse_addr(fd, 1))) {
     goto error;
   }
 
@@ -386,8 +393,12 @@ static int connect_client(struct sockaddr *addr, socklen_t len) {
     goto error;
   }
 
-  if (!grpc_set_socket_cloexec(fd, 1) || !grpc_set_socket_low_latency(fd, 1)) {
-    gpr_log(GPR_ERROR, "Failed to configure socket");
+  if (!GRPC_LOG_IF_ERROR("Failed to set connecting socket cloexec",
+                         grpc_set_socket_cloexec(fd, 1))) {
+    goto error;
+  }
+  if (!GRPC_LOG_IF_ERROR("Failed to set connecting socket low latency",
+                         grpc_set_socket_low_latency(fd, 1))) {
     goto error;
   }
 
diff --git a/test/core/profiling/mark_timings.stp b/test/core/profiling/mark_timings.stp
index 0c0a417faf..e96edbc149 100644
--- a/test/core/profiling/mark_timings.stp
+++ b/test/core/profiling/mark_timings.stp
@@ -2,7 +2,7 @@
  * probe definition.
  *
  * For a statically build binary, that'd be the name of the binary itself.
- * For dinamically built ones, point to the location of the libgprc.so being
+ * For dynamically built ones, point to the location of the libgrpc.so being
  * used. */
 
 global starts, times, times_per_tag
diff --git a/test/core/util/mock_endpoint.c b/test/core/util/mock_endpoint.c
index 28ff0642cc..bf6d85252a 100644
--- a/test/core/util/mock_endpoint.c
+++ b/test/core/util/mock_endpoint.c
@@ -37,6 +37,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
+#include "src/core/lib/iomgr/sockaddr.h"
 
 typedef struct grpc_mock_endpoint {
   grpc_endpoint base;
@@ -105,6 +106,8 @@ static grpc_resource_user *me_get_resource_user(grpc_endpoint *ep) {
   return m->resource_user;
 }
 
+static int me_get_fd(grpc_endpoint *ep) { return -1; }
+
 static grpc_workqueue *me_get_workqueue(grpc_endpoint *ep) { return NULL; }
 
 static const grpc_endpoint_vtable vtable = {
@@ -117,6 +120,7 @@ static const grpc_endpoint_vtable vtable = {
     me_destroy,
     me_get_resource_user,
     me_get_peer,
+    me_get_fd,
 };
 
 grpc_endpoint *grpc_mock_endpoint_create(void (*on_write)(grpc_slice slice),
diff --git a/test/core/util/passthru_endpoint.c b/test/core/util/passthru_endpoint.c
index 8b323d1e45..b3405f02e9 100644
--- a/test/core/util/passthru_endpoint.c
+++ b/test/core/util/passthru_endpoint.c
@@ -37,6 +37,7 @@
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
+#include "src/core/lib/iomgr/sockaddr.h"
 
 typedef struct passthru_endpoint passthru_endpoint;
 
@@ -146,6 +147,8 @@ static char *me_get_peer(grpc_endpoint *ep) {
   return gpr_strdup("fake:mock_endpoint");
 }
 
+static int me_get_fd(grpc_endpoint *ep) { return -1; }
+
 static grpc_workqueue *me_get_workqueue(grpc_endpoint *ep) { return NULL; }
 
 static grpc_resource_user *me_get_resource_user(grpc_endpoint *ep) {
@@ -163,6 +166,7 @@ static const grpc_endpoint_vtable vtable = {
     me_destroy,
     me_get_resource_user,
     me_get_peer,
+    me_get_fd,
 };
 
 static void half_init(half *m, passthru_endpoint *parent,