Merge branch 'grpclb_resolver_changes' into grpclb_resolver_changes2

38 files changed, 999 insertions, 675 deletions

diff --git a/src/core/ext/client_config/client_channel.c b/src/core/ext/client_config/client_channel.c
index 0343a5e254..1696c005c8 100644
--- a/src/core/ext/client_config/client_channel.c
+++ b/src/core/ext/client_config/client_channel.c
@@ -406,6 +406,8 @@ typedef struct client_channel_call_data {
   grpc_closure next_step;
 
   grpc_call_stack *owning_call;
+
+  grpc_linked_mdelem lb_token_mdelem;
 } call_data;
 
 static void add_waiting_locked(call_data *calld, grpc_transport_stream_op *op) {
@@ -578,9 +580,11 @@ static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     int r;
     GRPC_LB_POLICY_REF(lb_policy, "pick_subchannel");
     gpr_mu_unlock(&chand->mu);
-    r = grpc_lb_policy_pick(exec_ctx, lb_policy, calld->pollent,
-                            initial_metadata, initial_metadata_flags,
-                            connected_subchannel, on_ready);
+    const grpc_lb_policy_pick_args inputs = {calld->pollent, initial_metadata,
+                                             initial_metadata_flags,
+                                             &calld->lb_token_mdelem};
+    r = grpc_lb_policy_pick(exec_ctx, lb_policy, &inputs, connected_subchannel,
+                            NULL, on_ready);
     GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "pick_subchannel");
     GPR_TIMER_END("pick_subchannel", 0);
     return r;
diff --git a/src/core/ext/client_config/lb_policy.c b/src/core/ext/client_config/lb_policy.c
index 8b980b2cca..903563ef6b 100644
--- a/src/core/ext/client_config/lb_policy.c
+++ b/src/core/ext/client_config/lb_policy.c
@@ -100,13 +100,11 @@ void grpc_lb_policy_weak_unref(grpc_exec_ctx *exec_ctx,
 }
 
 int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-                        grpc_polling_entity *pollent,
-                        grpc_metadata_batch *initial_metadata,
-                        uint32_t initial_metadata_flags,
-                        grpc_connected_subchannel **target,
+                        const grpc_lb_policy_pick_args *pick_args,
+                        grpc_connected_subchannel **target, void **user_data,
                         grpc_closure *on_complete) {
-  return policy->vtable->pick(exec_ctx, policy, pollent, initial_metadata,
-                              initial_metadata_flags, target, on_complete);
+  return policy->vtable->pick(exec_ctx, policy, pick_args, target, user_data,
+                              on_complete);
 }
 
 void grpc_lb_policy_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
diff --git a/src/core/ext/client_config/lb_policy.h b/src/core/ext/client_config/lb_policy.h
index a2f5446fc6..37c93d707c 100644
--- a/src/core/ext/client_config/lb_policy.h
+++ b/src/core/ext/client_config/lb_policy.h
@@ -53,23 +53,38 @@ struct grpc_lb_policy {
   grpc_pollset_set *interested_parties;
 };
 
+/** Extra arguments for an LB pick */
+typedef struct grpc_lb_policy_pick_args {
+  /** Parties interested in the pick's progress */
+  grpc_polling_entity *pollent;
+  /** Initial metadata associated with the picking call. */
+  grpc_metadata_batch *initial_metadata;
+  /** See \a GRPC_INITIAL_METADATA_* in grpc_types.h */
+  uint32_t initial_metadata_flags;
+  /** Storage for LB token in \a initial_metadata, or NULL if not used */
+  grpc_linked_mdelem *lb_token_mdelem_storage;
+} grpc_lb_policy_pick_args;
+
 struct grpc_lb_policy_vtable {
   void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy);
-
   void (*shutdown)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy);
 
-  /** implement grpc_lb_policy_pick */
+  /** \see grpc_lb_policy_pick */
   int (*pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-              grpc_polling_entity *pollent,
-              grpc_metadata_batch *initial_metadata,
-              uint32_t initial_metadata_flags,
-              grpc_connected_subchannel **target, grpc_closure *on_complete);
+              const grpc_lb_policy_pick_args *pick_args,
+              grpc_connected_subchannel **target, void **user_data,
+              grpc_closure *on_complete);
+
+  /** \see grpc_lb_policy_cancel_pick */
   void (*cancel_pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
                       grpc_connected_subchannel **target);
+
+  /** \see grpc_lb_policy_cancel_picks */
   void (*cancel_picks)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
                        uint32_t initial_metadata_flags_mask,
                        uint32_t initial_metadata_flags_eq);
 
+  /** \see grpc_lb_policy_ping_one */
   void (*ping_one)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
                    grpc_closure *closure);
 
@@ -83,8 +98,7 @@ struct grpc_lb_policy_vtable {
 
   /** call notify when the connectivity state of a channel changes from *state.
       Updates *state with the new state of the policy. Calling with a NULL \a
-      state cancels the subscription.
-      */
+      state cancels the subscription.  */
   void (*notify_on_state_change)(grpc_exec_ctx *exec_ctx,
                                  grpc_lb_policy *policy,
                                  grpc_connectivity_state *state,
@@ -124,26 +138,34 @@ void grpc_lb_policy_weak_unref(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy);
 void grpc_lb_policy_init(grpc_lb_policy *policy,
                          const grpc_lb_policy_vtable *vtable);
 
-/** Given initial metadata in \a initial_metadata, find an appropriate
-    target for this rpc, and 'return' it by calling \a on_complete after setting
-    \a target.
-    Picking can be asynchronous. Any IO should be done under \a pollent. */
+/** Find an appropriate target for this call, based on \a pick_args.
+    Picking can be synchronous or asynchronous. In the synchronous case, when a
+    pick is readily available, it'll be returned in \a target and a non-zero
+    value will be returned.
+    In the asynchronous case, zero is returned and \a on_complete will be called
+    once \a target and \a user_data have been set. Any IO should be done under
+    \a pick_args->pollent. The opaque \a user_data output argument corresponds
+    to information that may need be propagated from the LB policy. It may be
+    NULL. Errors are signaled by receiving a NULL \a *target. */
 int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
-                        grpc_polling_entity *pollent,
-                        grpc_metadata_batch *initial_metadata,
-                        uint32_t initial_metadata_flags,
-                        grpc_connected_subchannel **target,
+                        const grpc_lb_policy_pick_args *pick_args,
+                        grpc_connected_subchannel **target, void **user_data,
                         grpc_closure *on_complete);
 
+/** Perform a connected subchannel ping (see \a grpc_connected_subchannel_ping)
+    against one of the connected subchannels managed by \a policy. */
 void grpc_lb_policy_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
                              grpc_closure *closure);
 
+/** Cancel picks for \a target.
+    The \a on_complete callback of the pending picks will be invoked with \a
+    *target set to NULL. */
 void grpc_lb_policy_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
                                 grpc_connected_subchannel **target);
 
-/** Cancel all pending picks which have:
-    (initial_metadata_flags & initial_metadata_flags_mask) ==
-         initial_metadata_flags_eq */
+/** Cancel all pending picks for which their \a initial_metadata_flags (as given
+    in the call to \a grpc_lb_policy_pick) matches \a initial_metadata_flags_eq
+    when AND'd with \a initial_metadata_flags_mask */
 void grpc_lb_policy_cancel_picks(grpc_exec_ctx *exec_ctx,
                                  grpc_lb_policy *policy,
                                  uint32_t initial_metadata_flags_mask,
diff --git a/src/core/ext/client_config/lb_policy_factory.c b/src/core/ext/client_config/lb_policy_factory.c
index 70e46ef3cf..2737f141c4 100644
--- a/src/core/ext/client_config/lb_policy_factory.c
+++ b/src/core/ext/client_config/lb_policy_factory.c
@@ -31,8 +31,61 @@
  *
  */
 
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+
 #include "src/core/ext/client_config/lb_policy_factory.h"
 
+grpc_lb_addresses* grpc_lb_addresses_create(size_t num_addresses) {
+  grpc_lb_addresses* addresses = gpr_malloc(sizeof(grpc_lb_addresses));
+  addresses->num_addresses = num_addresses;
+  const size_t addresses_size = sizeof(grpc_lb_address) * num_addresses;
+  addresses->addresses = gpr_malloc(addresses_size);
+  memset(addresses->addresses, 0, addresses_size);
+  return addresses;
+}
+
+grpc_lb_addresses *grpc_lb_addresses_copy(grpc_lb_addresses* addresses,
+                                          void *(*user_data_copy)(void *)) {
+  grpc_lb_addresses* new = grpc_lb_addresses_create(addresses->num_addresses);
+  memcpy(new->addresses, addresses->addresses,
+         sizeof(grpc_address) * addresses->num_addresses);
+  for (size_t i = 0; i < addresses->num_addresses; ++i) {
+    new->addresses[i].balancer_name =
+        gpr_strdup(new->addresses[i].balancer_name);
+    if (user_data_copy != NULL) {
+      new->addresses[i].user_data =
+          user_data_copy(new->addresses[i].user_data);
+    }
+  }
+  return new;
+}
+
+void grpc_lb_addresses_set_address(grpc_lb_addresses* addresses, size_t index,
+                                   void* address, size_t address_len,
+                                   bool is_balancer, char* balancer_name,
+                                   void* user_data) {
+  GPR_ASSERT(index < addresses->num_addresses);
+  grpc_lb_address* target = &addresses->addresses[index];
+  memcpy(target->address.addr, address, address_len);
+  target->address.len = address_len;
+  target->is_balancer = is_balancer;
+  target->balancer_name = balancer_name;
+  target->user_data = user_data;
+}
+
+void grpc_lb_addresses_destroy(grpc_lb_addresses* addresses,
+                               void (*user_data_destroy)(void*)) {
+  for (size_t i = 0; i < addresses->num_addresses; ++i) {
+    gpr_free(addresses->addresses[i].balancer_name);
+    if (user_data_destroy != NULL)
+      user_data_destroy(addresses->addresses[i].user_data);
+  }
+  gpr_free(addresses->addresses);
+  gpr_free(addresses);
+}
+
 void grpc_lb_policy_factory_ref(grpc_lb_policy_factory* factory) {
   factory->vtable->ref(factory);
 }
diff --git a/src/core/ext/client_config/lb_policy_factory.h b/src/core/ext/client_config/lb_policy_factory.h
index e2364c31a8..70dadff0ab 100644
--- a/src/core/ext/client_config/lb_policy_factory.h
+++ b/src/core/ext/client_config/lb_policy_factory.h
@@ -47,10 +47,50 @@ struct grpc_lb_policy_factory {
   const grpc_lb_policy_factory_vtable *vtable;
 };
 
-// TODO(roth, ctiller): Consider replacing this struct with
-// grpc_channel_args.  See comment in resolver_result.h for details.
+/** A resolved address alongside any LB related information associated with it.
+ * \a user_data, if not NULL, contains opaque data meant to be consumed by the
+ * gRPC LB policy. Note that no all LB policies support \a user_data as input.
+ * Those who don't will simply ignore it and will correspondingly return NULL in
+ * their namesake pick() output argument. */
+typedef struct grpc_lb_address {
+  grpc_resolved_address address;
+  bool is_balancer;
+  char *balancer_name; /* For secure naming. */
+  void *user_data;
+} grpc_lb_address;
+
+typedef struct grpc_lb_addresses {
+  size_t num_addresses;
+  grpc_lb_address *addresses;
+} grpc_lb_addresses;
+
+/** Returns a grpc_addresses struct with enough space for
+ * \a num_addresses addresses. */
+grpc_lb_addresses *grpc_lb_addresses_create(size_t num_addresses);
+
+/** Creates a copy of \a addresses.  If \a user_data_copy is not NULL,
+ * it will be invoked to copy the \a user_data field of each address. */
+grpc_lb_addresses *grpc_lb_addresses_copy(grpc_lb_addresses* addresses,
+                                          void *(*user_data_copy)(void *));
+
+/** Sets the value of the address at index \a index of \a addresses.
+ * \a address is a socket address of length \a address_len.
+ * Takes ownership of \a balancer_name. */
+void grpc_lb_addresses_set_address(grpc_lb_addresses *addresses, size_t index,
+                                   void *address, size_t address_len,
+                                   bool is_balancer, char *balancer_name,
+                                   void *user_data);
+
+/** Destroys \a addresses.  If \a user_data_destroy is not NULL, it will
+ * be invoked to destroy the \a user_data field of each address. */
+void grpc_lb_addresses_destroy(grpc_lb_addresses *addresses,
+                               void (*user_data_destroy)(void *));
+
+/** Arguments passed to LB policies. */
+/* TODO(roth, ctiller): Consider replacing this struct with
+   grpc_channel_args.  See comment in resolver_result.h for details. */
 typedef struct grpc_lb_policy_args {
-  grpc_addresses *addresses;
+  grpc_lb_addresses *addresses;
   grpc_client_channel_factory *client_channel_factory;
   grpc_channel_args *additional_args;
 } grpc_lb_policy_args;
diff --git a/src/core/ext/client_config/resolver_result.c b/src/core/ext/client_config/resolver_result.c
index f97dc467e1..c07bd8b4da 100644
--- a/src/core/ext/client_config/resolver_result.c
+++ b/src/core/ext/client_config/resolver_result.c
@@ -38,37 +38,6 @@
 
 #include "src/core/lib/channel/channel_args.h"
 
-grpc_addresses* grpc_addresses_create(size_t num_addresses) {
-  grpc_addresses* addresses = gpr_malloc(sizeof(grpc_addresses));
-  addresses->num_addresses = num_addresses;
-  const size_t addresses_size = sizeof(grpc_address) * num_addresses;
-  addresses->addresses = gpr_malloc(addresses_size);
-  memset(addresses->addresses, 0, addresses_size);
-  return addresses;
-}
-
-grpc_addresses* grpc_addresses_copy(grpc_addresses* addresses) {
-  grpc_addresses* new = grpc_addresses_create(addresses->num_addresses);
-  memcpy(new->addresses, addresses->addresses,
-         sizeof(grpc_address) * addresses->num_addresses);
-  return new;
-}
-
-void grpc_addresses_set_address(grpc_addresses* addresses, size_t index,
-                                void* address, size_t address_len,
-                                bool is_balancer) {
-  GPR_ASSERT(index < addresses->num_addresses);
-  grpc_address* target = &addresses->addresses[index];
-  memcpy(target->address.addr, address, address_len);
-  target->address.len = address_len;
-  target->is_balancer = is_balancer;
-}
-
-void grpc_addresses_destroy(grpc_addresses* addresses) {
-  gpr_free(addresses->addresses);
-  gpr_free(addresses);
-}
-
 struct grpc_resolver_result {
   gpr_refcount refs;
   grpc_addresses* addresses;
diff --git a/src/core/ext/client_config/resolver_result.h b/src/core/ext/client_config/resolver_result.h
index 821208f709..e28a2f5bf3 100644
--- a/src/core/ext/client_config/resolver_result.h
+++ b/src/core/ext/client_config/resolver_result.h
@@ -32,9 +32,7 @@
 #ifndef GRPC_CORE_EXT_CLIENT_CONFIG_RESOLVER_RESULT_H
 #define GRPC_CORE_EXT_CLIENT_CONFIG_RESOLVER_RESULT_H
 
-#include <stdbool.h>
-
-#include "src/core/ext/client_config/lb_policy.h"
+#include "src/core/ext/client_config/lb_policy_factory.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 
 // TODO(roth, ctiller): In the long term, we are considering replacing
@@ -47,35 +45,12 @@
 // grpc_channel_args such to a hash table or AVL or some other data
 // structure that does not require linear search to find keys.
 
-/// Used to represent addresses returned by the resolver.
-typedef struct grpc_address {
-  grpc_resolved_address address;
-  bool is_balancer;
-} grpc_address;
-
-typedef struct grpc_addresses {
-  size_t num_addresses;
-  grpc_address* addresses;
-} grpc_addresses;
-
-/// Returns a grpc_addresses struct with enough space for
-/// \a num_addresses addresses.
-grpc_addresses* grpc_addresses_create(size_t num_addresses);
-
-grpc_addresses* grpc_addresses_copy(grpc_addresses* addresses);
-
-void grpc_addresses_set_address(grpc_addresses* addresses, size_t index,
-                                void* address, size_t address_len,
-                                bool is_balancer);
-
-void grpc_addresses_destroy(grpc_addresses* addresses);
-
 /// Results reported from a grpc_resolver.
 typedef struct grpc_resolver_result grpc_resolver_result;
 
 /// Takes ownership of \a addresses and \a lb_policy_args.
 grpc_resolver_result* grpc_resolver_result_create(
-    grpc_addresses* addresses, const char* lb_policy_name,
+    grpc_lb_addresses* addresses, const char* lb_policy_name,
     grpc_channel_args* lb_policy_args);
 void grpc_resolver_result_ref(grpc_resolver_result* result);
 void grpc_resolver_result_unref(grpc_exec_ctx* exec_ctx,
diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c
index 61721a4a9e..999122fbe0 100644
--- a/src/core/ext/lb_policy/grpclb/grpclb.c
+++ b/src/core/ext/lb_policy/grpclb/grpclb.c
@@ -76,9 +76,9 @@
  *    operations in progress over the old RR instance. This is done by
  *    decreasing the reference count on the old policy. The moment no more
  *    references are held on the old RR policy, it'll be destroyed and \a
- *    rr_connectivity_changed notified with a \a GRPC_CHANNEL_SHUTDOWN state.
- *    At this point we can transition to a new RR instance safely, which is done
- *    once again via \a rr_handover().
+ *    glb_rr_connectivity_changed notified with a \a GRPC_CHANNEL_SHUTDOWN
+ *    state. At this point we can transition to a new RR instance safely, which
+ *    is done once again via \a rr_handover().
  *
  *
  * Once a RR policy instance is in place (and getting updated as described),
@@ -96,6 +96,8 @@
  * - Implement LB service forwarding (point 2c. in the doc's diagram).
  */
 
+#include <errno.h>
+
 #include <string.h>
 
 #include <grpc/byte_buffer_reader.h>
@@ -105,22 +107,50 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/ext/client_config/client_channel_factory.h"
+#include "src/core/ext/client_config/lb_policy_factory.h"
 #include "src/core/ext/client_config/lb_policy_registry.h"
 #include "src/core/ext/client_config/parse_address.h"
 #include "src/core/ext/lb_policy/grpclb/grpclb.h"
 #include "src/core/ext/lb_policy/grpclb/load_balancer_api.h"
+#include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/surface/channel.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 int grpc_lb_glb_trace = 0;
 
+/* add lb_token of selected subchannel (address) to the call's initial
+ * metadata */
+static void initial_metadata_add_lb_token(
+    grpc_metadata_batch *initial_metadata,
+    grpc_linked_mdelem *lb_token_mdelem_storage, grpc_mdelem *lb_token) {
+  GPR_ASSERT(lb_token_mdelem_storage != NULL);
+  GPR_ASSERT(lb_token != NULL);
+  grpc_metadata_batch_add_tail(initial_metadata, lb_token_mdelem_storage,
+                               lb_token);
+}
+
 typedef struct wrapped_rr_closure_arg {
   /* the original closure. Usually a on_complete/notify cb for pick() and ping()
    * calls against the internal RR instance, respectively. */
   grpc_closure *wrapped_closure;
 
+  /* the pick's initial metadata, kept in order to append the LB token for the
+   * pick */
+  grpc_metadata_batch *initial_metadata;
+
+  /* the picked target, used to determine which LB token to add to the pick's
+   * initial metadata */
+  grpc_connected_subchannel **target;
+
+  /* the LB token associated with the pick */
+  grpc_mdelem *lb_token;
+
+  /* storage for the lb token initial metadata mdelem */
+  grpc_linked_mdelem *lb_token_mdelem_storage;
+
   /* The RR instance related to the closure */
   grpc_lb_policy *rr_policy;
 
@@ -141,9 +171,20 @@ static void wrapped_rr_closure(grpc_exec_ctx *exec_ctx, void *arg,
               (intptr_t)wc_arg->rr_policy);
     }
     GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "wrapped_rr_closure");
+
+    /* if target is NULL, no pick has been made by the RR policy (eg, all
+     * addresses failed to connect). There won't be any user_data/token
+     * available */
+    if (wc_arg->target != NULL) {
+      initial_metadata_add_lb_token(wc_arg->initial_metadata,
+                                    wc_arg->lb_token_mdelem_storage,
+                                    GRPC_MDELEM_REF(wc_arg->lb_token));
+    }
   }
   GPR_ASSERT(wc_arg->wrapped_closure != NULL);
-  grpc_exec_ctx_sched(exec_ctx, wc_arg->wrapped_closure, error, NULL);
+
+  grpc_exec_ctx_sched(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_REF(error),
+                      NULL);
   gpr_free(wc_arg->owning_pending_node);
 }
 
@@ -164,6 +205,9 @@ typedef struct pending_pick {
   /* the initial metadata for the pick. See grpc_lb_policy_pick() */
   grpc_metadata_batch *initial_metadata;
 
+  /* storage for the lb token initial metadata mdelem */
+  grpc_linked_mdelem *lb_token_mdelem_storage;
+
   /* bitmask passed to pick() and used for selective cancelling. See
    * grpc_lb_policy_cancel_picks() */
   uint32_t initial_metadata_flags;
@@ -180,20 +224,24 @@ typedef struct pending_pick {
   wrapped_rr_closure_arg wrapped_on_complete_arg;
 } pending_pick;
 
-static void add_pending_pick(pending_pick **root, grpc_polling_entity *pollent,
-                             grpc_metadata_batch *initial_metadata,
-                             uint32_t initial_metadata_flags,
+static void add_pending_pick(pending_pick **root,
+                             const grpc_lb_policy_pick_args *pick_args,
                              grpc_connected_subchannel **target,
                              grpc_closure *on_complete) {
   pending_pick *pp = gpr_malloc(sizeof(*pp));
   memset(pp, 0, sizeof(pending_pick));
   memset(&pp->wrapped_on_complete_arg, 0, sizeof(wrapped_rr_closure_arg));
   pp->next = *root;
-  pp->pollent = pollent;
+  pp->pollent = pick_args->pollent;
   pp->target = target;
-  pp->initial_metadata = initial_metadata;
-  pp->initial_metadata_flags = initial_metadata_flags;
+  pp->initial_metadata = pick_args->initial_metadata;
+  pp->initial_metadata_flags = pick_args->initial_metadata_flags;
+  pp->lb_token_mdelem_storage = pick_args->lb_token_mdelem_storage;
   pp->wrapped_on_complete_arg.wrapped_closure = on_complete;
+  pp->wrapped_on_complete_arg.target = target;
+  pp->wrapped_on_complete_arg.initial_metadata = pick_args->initial_metadata;
+  pp->wrapped_on_complete_arg.lb_token_mdelem_storage =
+      pick_args->lb_token_mdelem_storage;
   grpc_closure_init(&pp->wrapped_on_complete, wrapped_rr_closure,
                     &pp->wrapped_on_complete_arg);
   *root = pp;
@@ -252,6 +300,9 @@ typedef struct glb_lb_policy {
    * response has arrived. */
   grpc_grpclb_serverlist *serverlist;
 
+  /** addresses from \a serverlist */
+  grpc_lb_addresses *addresses;
+
   /** list of picks that are waiting on RR's policy connectivity */
   pending_pick *pending_picks;
 
@@ -279,58 +330,133 @@ struct rr_connectivity_data {
   glb_lb_policy *glb_policy;
 };
 
-static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx,
-                                 const grpc_grpclb_serverlist *serverlist,
-                                 glb_lb_policy *glb_policy) {
-  /* TODO(dgq): support mixed ip version */
-  GPR_ASSERT(serverlist != NULL && serverlist->num_servers > 0);
-  char **host_ports = gpr_malloc(sizeof(char *) * serverlist->num_servers);
-  for (size_t i = 0; i < serverlist->num_servers; ++i) {
-    gpr_join_host_port(&host_ports[i], serverlist->servers[i]->ip_address,
-                       serverlist->servers[i]->port);
+static bool is_server_valid(const grpc_grpclb_server *server, size_t idx,
+                            bool log) {
+  const grpc_grpclb_ip_address *ip = &server->ip_address;
+  if (server->port >> 16 != 0) {
+    if (log) {
+      gpr_log(GPR_ERROR,
+              "Invalid port '%d' at index %zu of serverlist. Ignoring.",
+              server->port, idx);
+    }
+    return false;
   }
 
-  size_t uri_path_len;
-  char *concat_ipports = gpr_strjoin_sep(
-      (const char **)host_ports, serverlist->num_servers, ",", &uri_path_len);
+  if (ip->size != 4 && ip->size != 16) {
+    if (log) {
+      gpr_log(GPR_ERROR,
+              "Expected IP to be 4 or 16 bytes, got %d at index %zu of "
+              "serverlist. Ignoring",
+              ip->size, idx);
+    }
+    return false;
+  }
+  return true;
+}
 
-  grpc_lb_policy_args args;
-  args.client_channel_factory = glb_policy->cc_factory;
-  args.addresses = grpc_addresses_create(serverlist->num_servers);
-  size_t out_addrs_idx = 0;
+/* populate \a addresses according to \a serverlist. Returns the number of
+ * addresses successfully parsed and added to \a addresses */
+static grpc_lb_addresses *process_serverlist(
+    const grpc_grpclb_serverlist *serverlist) {
+  size_t num_valid = 0;
+  /* first pass: count how many are valid in order to allocate the necessary
+   * memory in a single block */
   for (size_t i = 0; i < serverlist->num_servers; ++i) {
-    grpc_uri uri;
-    struct sockaddr_storage sa;
-    size_t sa_len;
-    uri.path = host_ports[i];
-    if (parse_ipv4(&uri, &sa, &sa_len)) { /* TODO(dgq): add support for ipv6 */
-      /* These are, of course, actually balancer addresses. However, we
-       * want the round_robin LB policy to treat them as normal backend
-       * addresses, since we don't need to talk to balancers in order to
-       * find the balancers themselves, so we set is_balancer=false. */
-      grpc_addresses_set_address(args.addresses, out_addrs_idx, &sa, sa_len,
-                                 false /* is_balancer */);
-      ++out_addrs_idx;
+    if (is_server_valid(serverlist->servers[i], i, true)) ++num_valid;
+  }
+  if (num_valid == 0) return NULL;
+
+  grpc_lb_addresses *lb_addresses = grpc_lb_addresses_create(num_valid);
+
+  /* second pass: actually populate the addresses and LB tokens (aka user data
+   * to the outside world) to be read by the RR policy during its creation.
+   * Given that the validity tests are very cheap, they are performed again
+   * instead of marking the valid ones during the first pass, as this would
+   * incurr in an allocation due to the arbitrary number of server */
+  size_t addr_idx = 0;
+  for (size_t sl_idx = 0; sl_idx < serverlist->num_servers; ++sl_idx) {
+    GPR_ASSERT(addr_idx < num_valid);
+    const grpc_grpclb_server *server = serverlist->servers[sl_idx];
+    if (!is_server_valid(serverlist->servers[sl_idx], sl_idx, false)) continue;
+
+    /* address processing */
+    const uint16_t netorder_port = htons((uint16_t)server->port);
+    /* the addresses are given in binary format (a in(6)_addr struct) in
+     * server->ip_address.bytes. */
+    const grpc_grpclb_ip_address *ip = &server->ip_address;
+    grpc_resolved_address addr;
+    memset(&addr, 0, sizeof(addr));
+    if (ip->size == 4) {
+      addr.len = sizeof(struct sockaddr_in);
+      struct sockaddr_in *addr4 = (struct sockaddr_in *)&addr.addr;
+      addr4->sin_family = AF_INET;
+      memcpy(&addr4->sin_addr, ip->bytes, ip->size);
+      addr4->sin_port = netorder_port;
+    } else if (ip->size == 16) {
+      addr.len = sizeof(struct sockaddr_in6);
+      struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *)&addr.addr;
+      addr6->sin6_family = AF_INET;
+      memcpy(&addr6->sin6_addr, ip->bytes, ip->size);
+      addr6->sin6_port = netorder_port;
+    }
+
+    /* lb token processing */
+    void *user_data;
+    if (server->has_load_balance_token) {
+      const size_t lb_token_size =
+          GPR_ARRAY_SIZE(server->load_balance_token) - 1;
+      grpc_mdstr *lb_token_mdstr = grpc_mdstr_from_buffer(
+          (uint8_t *)server->load_balance_token, lb_token_size);
+      user_data = grpc_mdelem_from_metadata_strings(
+          GRPC_MDSTR_LOAD_REPORTING_INITIAL, lb_token_mdstr);
     } else {
-      gpr_log(GPR_ERROR, "Invalid LB service address '%s', ignoring.",
-              host_ports[i]);
+      gpr_log(GPR_ERROR,
+              "Missing LB token for backend address '%s'. The empty token will "
+              "be used instead",
+              grpc_sockaddr_to_uri((struct sockaddr *)&addr.addr));
+      user_data = GRPC_MDELEM_LOAD_REPORTING_INITIAL_EMPTY;
     }
+
+    grpc_lb_addresses_set_address(lb_addresses, addr_idx, &addr.addr, addr.len,
+                                  false /* is_balancer */,
+                                  NULL /* balancer_name */, user_data);
+    ++addr_idx;
   }
+  GPR_ASSERT(addr_idx == num_valid);
+
+  return lb_addresses;
+}
+
+/* A plugin for grpc_lb_addresses_destroy that unrefs the LB token metadata. */
+static void lb_token_destroy(void *token) {
+  if (token != NULL) GRPC_MDELEM_UNREF(token);
+}
+
+static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx,
+                                 const grpc_grpclb_serverlist *serverlist,
+                                 glb_lb_policy *glb_policy) {
+  GPR_ASSERT(serverlist != NULL && serverlist->num_servers > 0);
+
+  grpc_lb_policy_args args;
+  memset(&args, 0, sizeof(args));
+  args.client_channel_factory = glb_policy->cc_factory;
+  args.addresses = process_serverlist(serverlist);
 
   grpc_lb_policy *rr = grpc_lb_policy_create(exec_ctx, "round_robin", &args);
 
-  gpr_free(concat_ipports);
-  for (size_t i = 0; i < serverlist->num_servers; i++) {
-    gpr_free(host_ports[i]);
+  if (glb_policy->addresses != NULL) {
+    /* dispose of the previous version */
+    grpc_lb_addresses_destroy(glb_policy->addresses, lb_token_destroy);
   }
-  gpr_free(host_ports);
-  grpc_addresses_destroy(args.addresses);
+  glb_policy->addresses = args.addresses;
+
   return rr;
 }
 
 static void rr_handover(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy,
                         grpc_error *error) {
-  GRPC_ERROR_REF(error);
+  GPR_ASSERT(glb_policy->serverlist != NULL &&
+             glb_policy->serverlist->num_servers > 0);
   glb_policy->rr_policy =
       create_rr(exec_ctx, glb_policy->serverlist, glb_policy);
 
@@ -345,8 +471,8 @@ static void rr_handover(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy,
       exec_ctx, glb_policy->rr_policy, &glb_policy->rr_connectivity->state,
       &glb_policy->rr_connectivity->on_change);
   grpc_connectivity_state_set(exec_ctx, &glb_policy->state_tracker,
-                              glb_policy->rr_connectivity->state, error,
-                              "rr_handover");
+                              glb_policy->rr_connectivity->state,
+                              GRPC_ERROR_REF(error), "rr_handover");
   grpc_lb_policy_exit_idle(exec_ctx, glb_policy->rr_policy);
 
   /* flush pending ops */
@@ -359,9 +485,12 @@ static void rr_handover(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy,
       gpr_log(GPR_INFO, "Pending pick about to PICK from 0x%" PRIxPTR "",
               (intptr_t)glb_policy->rr_policy);
     }
-    grpc_lb_policy_pick(exec_ctx, glb_policy->rr_policy, pp->pollent,
-                        pp->initial_metadata, pp->initial_metadata_flags,
-                        pp->target, &pp->wrapped_on_complete);
+    const grpc_lb_policy_pick_args pick_args = {
+        pp->pollent, pp->initial_metadata, pp->initial_metadata_flags,
+        pp->lb_token_mdelem_storage};
+    grpc_lb_policy_pick(exec_ctx, glb_policy->rr_policy, &pick_args, pp->target,
+                        (void **)&pp->wrapped_on_complete_arg.lb_token,
+                        &pp->wrapped_on_complete);
     pp->wrapped_on_complete_arg.owning_pending_node = pp;
   }
 
@@ -378,13 +507,13 @@ static void rr_handover(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy,
                             &pping->wrapped_notify);
     pping->wrapped_notify_arg.owning_pending_node = pping;
   }
-  GRPC_ERROR_UNREF(error);
 }
 
-static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
-                                    grpc_error *error) {
+static void glb_rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
+                                        grpc_error *error) {
   rr_connectivity_data *rr_conn_data = arg;
   glb_lb_policy *glb_policy = rr_conn_data->glb_policy;
+
   if (rr_conn_data->state == GRPC_CHANNEL_SHUTDOWN) {
     if (glb_policy->serverlist != NULL) {
       /* a RR policy is shutting down but there's a serverlist available ->
@@ -398,8 +527,8 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
     if (error == GRPC_ERROR_NONE) {
       /* RR not shutting down. Mimic the RR's policy state */
       grpc_connectivity_state_set(exec_ctx, &glb_policy->state_tracker,
-                                  rr_conn_data->state, error,
-                                  "rr_connectivity_changed");
+                                  rr_conn_data->state, GRPC_ERROR_REF(error),
+                                  "glb_rr_connectivity_changed");
       /* resubscribe */
       grpc_lb_policy_notify_on_state_change(exec_ctx, glb_policy->rr_policy,
                                             &rr_conn_data->state,
@@ -408,7 +537,6 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
       gpr_free(rr_conn_data);
     }
   }
-  GRPC_ERROR_UNREF(error);
 }
 
 static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
@@ -431,27 +559,36 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
   memset(glb_policy, 0, sizeof(*glb_policy));
 
   /* All input addresses in args->addresses come from a resolver that claims
-   * they are LB services. It's the resolver's responsibility to make sure this
+   * they are LB services. It's the resolver's responsibility to make sure
+   * this
    * policy is only instantiated and used in that case.
    *
    * Create a client channel over them to communicate with a LB service */
   glb_policy->cc_factory = args->client_channel_factory;
   GPR_ASSERT(glb_policy->cc_factory != NULL);
 
-  /* construct a target from the args->addresses, in the form
+  /* construct a target from the addresses in args, given in the form
    * ipvX://ip1:port1,ip2:port2,...
    * TODO(dgq): support mixed ip version */
   char **addr_strs = gpr_malloc(sizeof(char *) * num_grpclb_addrs);
-  addr_strs[0] = grpc_sockaddr_to_uri(
-      (const struct sockaddr *)&args->addresses->addresses[0].address.addr);
-  size_t addr_index = 1;
-  for (size_t i = 1; i < args->addresses->num_addresses; i++) {
+  size_t addr_index = 0;
+  for (size_t i = 0; i < args->addresses->num_addresses; i++) {
+    if (args->addresses->addresses[i].user_data != NULL) {
+      gpr_log(GPR_ERROR,
+              "This LB policy doesn't support user data. It will be ignored");
+    }
     if (args->addresses->addresses[i].is_balancer) {
-      GPR_ASSERT(grpc_sockaddr_to_string(
-                     &addr_strs[addr_index++],
-                     (const struct sockaddr *)&args->addresses->addresses[i]
-                         .address.addr,
-                     true) == 0);
+      if (addr_index == 0) {
+        addr_strs[addr_index++] = grpc_sockaddr_to_uri(
+            (const struct sockaddr *)&args->addresses->addresses[i]
+                .address.addr);
+      } else {
+        GPR_ASSERT(grpc_sockaddr_to_string(
+                       &addr_strs[addr_index++],
+                       (const struct sockaddr *)&args->addresses->addresses[i]
+                           .address.addr,
+                       true) == 0);
+      }
     }
   }
   size_t uri_path_len;
@@ -477,7 +614,7 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
   rr_connectivity_data *rr_connectivity =
       gpr_malloc(sizeof(rr_connectivity_data));
   memset(rr_connectivity, 0, sizeof(rr_connectivity_data));
-  grpc_closure_init(&rr_connectivity->on_change, rr_connectivity_changed,
+  grpc_closure_init(&rr_connectivity->on_change, glb_rr_connectivity_changed,
                     rr_connectivity);
   rr_connectivity->glb_policy = glb_policy;
   glb_policy->rr_connectivity = rr_connectivity;
@@ -500,6 +637,7 @@ static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
     grpc_grpclb_destroy_serverlist(glb_policy->serverlist);
   }
   gpr_mu_destroy(&glb_policy->mu);
+  grpc_lb_addresses_destroy(glb_policy->addresses, lb_token_destroy);
   gpr_free(glb_policy);
 }
 
@@ -560,7 +698,6 @@ static void glb_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
       *target = NULL;
       grpc_exec_ctx_sched(exec_ctx, &pp->wrapped_on_complete,
                           GRPC_ERROR_CANCELLED, NULL);
-      gpr_free(pp);
     } else {
       pp->next = glb_policy->pending_picks;
       glb_policy->pending_picks = pp;
@@ -590,7 +727,6 @@ static void glb_cancel_picks(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
           exec_ctx, pp->pollent, glb_policy->base.interested_parties);
       grpc_exec_ctx_sched(exec_ctx, &pp->wrapped_on_complete,
                           GRPC_ERROR_CANCELLED, NULL);
-      gpr_free(pp);
     } else {
       pp->next = glb_policy->pending_picks;
       glb_policy->pending_picks = pp;
@@ -617,12 +753,21 @@ static void glb_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
 }
 
 static int glb_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
-                    grpc_polling_entity *pollent,
-                    grpc_metadata_batch *initial_metadata,
-                    uint32_t initial_metadata_flags,
-                    grpc_connected_subchannel **target,
+                    const grpc_lb_policy_pick_args *pick_args,
+                    grpc_connected_subchannel **target, void **user_data,
                     grpc_closure *on_complete) {
   glb_lb_policy *glb_policy = (glb_lb_policy *)pol;
+
+  if (pick_args->lb_token_mdelem_storage == NULL) {
+    *target = NULL;
+    grpc_exec_ctx_sched(
+        exec_ctx, on_complete,
+        GRPC_ERROR_CREATE("No mdelem storage for the LB token. Load reporting "
+                          "won't work without it. Failing"),
+        NULL);
+    return 1;
+  }
+
   gpr_mu_lock(&glb_policy->mu);
   int r;
 
@@ -634,29 +779,36 @@ static int glb_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     GRPC_LB_POLICY_REF(glb_policy->rr_policy, "glb_pick");
     memset(&glb_policy->wc_arg, 0, sizeof(wrapped_rr_closure_arg));
     glb_policy->wc_arg.rr_policy = glb_policy->rr_policy;
+    glb_policy->wc_arg.target = target;
     glb_policy->wc_arg.wrapped_closure = on_complete;
+    glb_policy->wc_arg.lb_token_mdelem_storage =
+        pick_args->lb_token_mdelem_storage;
+    glb_policy->wc_arg.initial_metadata = pick_args->initial_metadata;
+    glb_policy->wc_arg.owning_pending_node = NULL;
     grpc_closure_init(&glb_policy->wrapped_on_complete, wrapped_rr_closure,
                       &glb_policy->wc_arg);
-    r = grpc_lb_policy_pick(exec_ctx, glb_policy->rr_policy, pollent,
-                            initial_metadata, initial_metadata_flags, target,
+
+    r = grpc_lb_policy_pick(exec_ctx, glb_policy->rr_policy, pick_args, target,
+                            (void **)&glb_policy->wc_arg.lb_token,
                             &glb_policy->wrapped_on_complete);
     if (r != 0) {
-      /* the call to grpc_lb_policy_pick has been sychronous. Unreffing the RR
-       * policy and notify the original callback */
-      glb_policy->wc_arg.wrapped_closure = NULL;
+      /* synchronous grpc_lb_policy_pick call. Unref the RR policy. */
       if (grpc_lb_glb_trace) {
         gpr_log(GPR_INFO, "Unreffing RR (0x%" PRIxPTR ")",
                 (intptr_t)glb_policy->wc_arg.rr_policy);
       }
       GRPC_LB_POLICY_UNREF(exec_ctx, glb_policy->wc_arg.rr_policy, "glb_pick");
-      grpc_exec_ctx_sched(exec_ctx, glb_policy->wc_arg.wrapped_closure,
-                          GRPC_ERROR_NONE, NULL);
+
+      /* add the load reporting initial metadata */
+      initial_metadata_add_lb_token(
+          pick_args->initial_metadata, pick_args->lb_token_mdelem_storage,
+          GRPC_MDELEM_REF(glb_policy->wc_arg.lb_token));
     }
   } else {
-    grpc_polling_entity_add_to_pollset_set(exec_ctx, pollent,
+    grpc_polling_entity_add_to_pollset_set(exec_ctx, pick_args->pollent,
                                            glb_policy->base.interested_parties);
-    add_pending_pick(&glb_policy->pending_picks, pollent, initial_metadata,
-                     initial_metadata_flags, target, on_complete);
+    add_pending_pick(&glb_policy->pending_picks, pick_args, target,
+                     on_complete);
 
     if (!glb_policy->started_picking) {
       start_picking(exec_ctx, glb_policy);
@@ -716,9 +868,6 @@ typedef struct lb_client_data {
   /* called once initial metadata's been sent */
   grpc_closure md_sent;
 
-  /* called once initial metadata's been received */
-  grpc_closure md_rcvd;
-
   /* called once the LoadBalanceRequest has been sent to the LB server. See
    * src/proto/grpc/.../load_balancer.proto */
   grpc_closure req_sent;
@@ -755,7 +904,6 @@ typedef struct lb_client_data {
 } lb_client_data;
 
 static void md_sent_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error);
-static void md_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error);
 static void req_sent_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error);
 static void res_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error);
 static void close_sent_cb(grpc_exec_ctx *exec_ctx, void *arg,
@@ -770,7 +918,6 @@ static lb_client_data *lb_client_data_create(glb_lb_policy *glb_policy) {
   gpr_mu_init(&lb_client->mu);
   grpc_closure_init(&lb_client->md_sent, md_sent_cb, lb_client);
 
-  grpc_closure_init(&lb_client->md_rcvd, md_recv_cb, lb_client);
   grpc_closure_init(&lb_client->req_sent, req_sent_cb, lb_client);
   grpc_closure_init(&lb_client->res_rcvd, res_recv_cb, lb_client);
   grpc_closure_init(&lb_client->close_sent, close_sent_cb, lb_client);
@@ -869,23 +1016,6 @@ static void md_sent_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   grpc_op ops[1];
   memset(ops, 0, sizeof(ops));
   grpc_op *op = ops;
-  op->op = GRPC_OP_RECV_INITIAL_METADATA;
-  op->data.recv_initial_metadata = &lb_client->initial_metadata_recv;
-  op->flags = 0;
-  op->reserved = NULL;
-  op++;
-  grpc_call_error call_error = grpc_call_start_batch_and_execute(
-      exec_ctx, lb_client->lb_call, ops, (size_t)(op - ops),
-      &lb_client->md_rcvd);
-  GPR_ASSERT(GRPC_CALL_OK == call_error);
-}
-
-static void md_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
-  lb_client_data *lb_client = arg;
-  GPR_ASSERT(lb_client->lb_call);
-  grpc_op ops[1];
-  memset(ops, 0, sizeof(ops));
-  grpc_op *op = ops;
 
   op->op = GRPC_OP_SEND_MESSAGE;
   op->data.send_message = lb_client->request_payload;
@@ -900,11 +1030,18 @@ static void md_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
 
 static void req_sent_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   lb_client_data *lb_client = arg;
+  GPR_ASSERT(lb_client->lb_call);
 
-  grpc_op ops[1];
+  grpc_op ops[2];
   memset(ops, 0, sizeof(ops));
   grpc_op *op = ops;
 
+  op->op = GRPC_OP_RECV_INITIAL_METADATA;
+  op->data.recv_initial_metadata = &lb_client->initial_metadata_recv;
+  op->flags = 0;
+  op->reserved = NULL;
+  op++;
+
   op->op = GRPC_OP_RECV_MESSAGE;
   op->data.recv_message = &lb_client->response_payload;
   op->flags = 0;
@@ -923,8 +1060,7 @@ static void res_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   grpc_op *op = ops;
   if (lb_client->response_payload != NULL) {
     /* Received data from the LB server. Look inside
-     * lb_client->response_payload, for
-     * a serverlist. */
+     * lb_client->response_payload, for a serverlist. */
     grpc_byte_buffer_reader bbr;
     grpc_byte_buffer_reader_init(&bbr, lb_client->response_payload);
     gpr_slice response_slice = grpc_byte_buffer_reader_readall(&bbr);
@@ -961,7 +1097,7 @@ static void res_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
         } else {
           /* unref the RR policy, eventually leading to its substitution with a
            * new one constructed from the received serverlist (see
-           * rr_connectivity_changed) */
+           * glb_rr_connectivity_changed) */
           GRPC_LB_POLICY_UNREF(exec_ctx, lb_client->glb_policy->rr_policy,
                                "serverlist_received");
         }
@@ -1024,8 +1160,8 @@ static void srv_status_rcvd_cb(grpc_exec_ctx *exec_ctx, void *arg,
             lb_client->status, lb_client->status_details,
             lb_client->status_details_capacity);
   }
-  /* TODO(dgq): deal with stream termination properly (fire up another one? fail
-   * the original call?) */
+  /* TODO(dgq): deal with stream termination properly (fire up another one?
+   * fail the original call?) */
 }
 
 /* Code wiring the policy with the rest of the core */
diff --git a/src/core/ext/lb_policy/grpclb/load_balancer_api.c b/src/core/ext/lb_policy/grpclb/load_balancer_api.c
index f4720a1345..a8881004a0 100644
--- a/src/core/ext/lb_policy/grpclb/load_balancer_api.c
+++ b/src/core/ext/lb_policy/grpclb/load_balancer_api.c
@@ -57,6 +57,7 @@ static bool decode_serverlist(pb_istream_t *stream, const pb_field_t *field,
   if (dec_arg->first_pass) { /* count how many server do we have */
     grpc_grpclb_server server;
     if (!pb_decode(stream, grpc_lb_v1_Server_fields, &server)) {
+      gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(stream));
       return false;
     }
     dec_arg->num_servers++;
@@ -69,6 +70,7 @@ static bool decode_serverlist(pb_istream_t *stream, const pb_field_t *field,
           gpr_malloc(sizeof(grpc_grpclb_server *) * dec_arg->num_servers);
     }
     if (!pb_decode(stream, grpc_lb_v1_Server_fields, server)) {
+      gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(stream));
       return false;
     }
     dec_arg->servers[dec_arg->decoding_idx++] = server;
@@ -118,6 +120,7 @@ grpc_grpclb_initial_response *grpc_grpclb_initial_response_parse(
   grpc_grpclb_response res;
   memset(&res, 0, sizeof(grpc_grpclb_response));
   if (!pb_decode(&stream, grpc_lb_v1_LoadBalanceResponse_fields, &res)) {
+    gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(&stream));
     return NULL;
   }
   grpc_grpclb_initial_response *initial_res =
@@ -145,6 +148,7 @@ grpc_grpclb_serverlist *grpc_grpclb_response_parse_serverlist(
   arg.first_pass = true;
   status = pb_decode(&stream, grpc_lb_v1_LoadBalanceResponse_fields, &res);
   if (!status) {
+    gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(&stream));
     return NULL;
   }
 
@@ -152,6 +156,7 @@ grpc_grpclb_serverlist *grpc_grpclb_response_parse_serverlist(
   status =
       pb_decode(&stream_at_start, grpc_lb_v1_LoadBalanceResponse_fields, &res);
   if (!status) {
+    gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(&stream));
     return NULL;
   }
 
diff --git a/src/core/ext/lb_policy/grpclb/load_balancer_api.h b/src/core/ext/lb_policy/grpclb/load_balancer_api.h
index 9726c87a37..c1e73d08ef 100644
--- a/src/core/ext/lb_policy/grpclb/load_balancer_api.h
+++ b/src/core/ext/lb_policy/grpclb/load_balancer_api.h
@@ -45,6 +45,7 @@ extern "C" {
 
 #define GRPC_GRPCLB_SERVICE_NAME_MAX_LENGTH 128
 
+typedef grpc_lb_v1_Server_ip_address_t grpc_grpclb_ip_address;
 typedef grpc_lb_v1_LoadBalanceRequest grpc_grpclb_request;
 typedef grpc_lb_v1_InitialLoadBalanceResponse grpc_grpclb_initial_response;
 typedef grpc_lb_v1_Server grpc_grpclb_server;
diff --git a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c
index 52e11c40bb..2676714175 100644
--- a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c
+++ b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c
@@ -31,10 +31,11 @@
  *
  */
 /* Automatically generated nanopb constant definitions */
-/* Generated by nanopb-0.3.5-dev */
+/* Generated by nanopb-0.3.7-dev */
 
 #include "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h"
 
+/* @@protoc_insertion_point(includes) */
 #if PB_PROTO_HEADER_VERSION != 30
 #error Regenerate this file with the current version of nanopb generator.
 #endif
@@ -72,7 +73,7 @@ const pb_field_t grpc_lb_v1_LoadBalanceResponse_fields[3] = {
 };
 
 const pb_field_t grpc_lb_v1_InitialLoadBalanceResponse_fields[3] = {
-    PB_FIELD(  2, STRING  , OPTIONAL, STATIC  , FIRST, grpc_lb_v1_InitialLoadBalanceResponse, load_balancer_delegate, load_balancer_delegate, 0),
+    PB_FIELD(  1, STRING  , OPTIONAL, STATIC  , FIRST, grpc_lb_v1_InitialLoadBalanceResponse, load_balancer_delegate, load_balancer_delegate, 0),
     PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval, load_balancer_delegate, &grpc_lb_v1_Duration_fields),
     PB_LAST_FIELD
 };
@@ -84,7 +85,7 @@ const pb_field_t grpc_lb_v1_ServerList_fields[3] = {
 };
 
 const pb_field_t grpc_lb_v1_Server_fields[5] = {
-    PB_FIELD(  1, STRING  , OPTIONAL, STATIC  , FIRST, grpc_lb_v1_Server, ip_address, ip_address, 0),
+    PB_FIELD(  1, BYTES   , OPTIONAL, STATIC  , FIRST, grpc_lb_v1_Server, ip_address, ip_address, 0),
     PB_FIELD(  2, INT32   , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_Server, port, ip_address, 0),
     PB_FIELD(  3, STRING  , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_Server, load_balance_token, port, 0),
     PB_FIELD(  4, BOOL    , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_Server, drop_request, load_balance_token, 0),
@@ -116,3 +117,4 @@ PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request)
 #endif
 
 
+/* @@protoc_insertion_point(eof) */
diff --git a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h
index 46fe588f72..4f1031ec7b 100644
--- a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h
+++ b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h
@@ -31,11 +31,12 @@
  *
  */
 /* Automatically generated nanopb header */
-/* Generated by nanopb-0.3.5-dev */
+/* Generated by nanopb-0.3.7-dev */
 
-#ifndef GRPC_CORE_EXT_LB_POLICY_GRPCLB_PROTO_GRPC_LB_V1_LOAD_BALANCER_PB_H
-#define GRPC_CORE_EXT_LB_POLICY_GRPCLB_PROTO_GRPC_LB_V1_LOAD_BALANCER_PB_H
+#ifndef PB_GRPC_LB_V1_LOAD_BALANCER_PB_H_INCLUDED
+#define PB_GRPC_LB_V1_LOAD_BALANCER_PB_H_INCLUDED
 #include "third_party/nanopb/pb.h"
+/* @@protoc_insertion_point(includes) */
 #if PB_PROTO_HEADER_VERSION != 30
 #error Regenerate this file with the current version of nanopb generator.
 #endif
@@ -52,6 +53,7 @@ typedef struct _grpc_lb_v1_ClientStats {
     int64_t client_rpc_errors;
     bool has_dropped_requests;
     int64_t dropped_requests;
+/* @@protoc_insertion_point(struct:grpc_lb_v1_ClientStats) */
 } grpc_lb_v1_ClientStats;
 
 typedef struct _grpc_lb_v1_Duration {
@@ -59,22 +61,26 @@ typedef struct _grpc_lb_v1_Duration {
     int64_t seconds;
     bool has_nanos;
     int32_t nanos;
+/* @@protoc_insertion_point(struct:grpc_lb_v1_Duration) */
 } grpc_lb_v1_Duration;
 
 typedef struct _grpc_lb_v1_InitialLoadBalanceRequest {
     bool has_name;
     char name[128];
+/* @@protoc_insertion_point(struct:grpc_lb_v1_InitialLoadBalanceRequest) */
 } grpc_lb_v1_InitialLoadBalanceRequest;
 
+typedef PB_BYTES_ARRAY_T(16) grpc_lb_v1_Server_ip_address_t;
 typedef struct _grpc_lb_v1_Server {
     bool has_ip_address;
-    char ip_address[46];
+    grpc_lb_v1_Server_ip_address_t ip_address;
     bool has_port;
     int32_t port;
     bool has_load_balance_token;
-    char load_balance_token[64];
+    char load_balance_token[65];
     bool has_drop_request;
     bool drop_request;
+/* @@protoc_insertion_point(struct:grpc_lb_v1_Server) */
 } grpc_lb_v1_Server;
 
 typedef struct _grpc_lb_v1_InitialLoadBalanceResponse {
@@ -82,6 +88,7 @@ typedef struct _grpc_lb_v1_InitialLoadBalanceResponse {
     char load_balancer_delegate[64];
     bool has_client_stats_report_interval;
     grpc_lb_v1_Duration client_stats_report_interval;
+/* @@protoc_insertion_point(struct:grpc_lb_v1_InitialLoadBalanceResponse) */
 } grpc_lb_v1_InitialLoadBalanceResponse;
 
 typedef struct _grpc_lb_v1_LoadBalanceRequest {
@@ -89,12 +96,14 @@ typedef struct _grpc_lb_v1_LoadBalanceRequest {
     grpc_lb_v1_InitialLoadBalanceRequest initial_request;
     bool has_client_stats;
     grpc_lb_v1_ClientStats client_stats;
+/* @@protoc_insertion_point(struct:grpc_lb_v1_LoadBalanceRequest) */
 } grpc_lb_v1_LoadBalanceRequest;
 
 typedef struct _grpc_lb_v1_ServerList {
     pb_callback_t servers;
     bool has_expiration_interval;
     grpc_lb_v1_Duration expiration_interval;
+/* @@protoc_insertion_point(struct:grpc_lb_v1_ServerList) */
 } grpc_lb_v1_ServerList;
 
 typedef struct _grpc_lb_v1_LoadBalanceResponse {
@@ -102,6 +111,7 @@ typedef struct _grpc_lb_v1_LoadBalanceResponse {
     grpc_lb_v1_InitialLoadBalanceResponse initial_response;
     bool has_server_list;
     grpc_lb_v1_ServerList server_list;
+/* @@protoc_insertion_point(struct:grpc_lb_v1_LoadBalanceResponse) */
 } grpc_lb_v1_LoadBalanceResponse;
 
 /* Default values for struct fields */
@@ -114,7 +124,7 @@ typedef struct _grpc_lb_v1_LoadBalanceResponse {
 #define grpc_lb_v1_LoadBalanceResponse_init_default {false, grpc_lb_v1_InitialLoadBalanceResponse_init_default, false, grpc_lb_v1_ServerList_init_default}
 #define grpc_lb_v1_InitialLoadBalanceResponse_init_default {false, "", false, grpc_lb_v1_Duration_init_default}
 #define grpc_lb_v1_ServerList_init_default       {{{NULL}, NULL}, false, grpc_lb_v1_Duration_init_default}
-#define grpc_lb_v1_Server_init_default           {false, "", false, 0, false, "", false, 0}
+#define grpc_lb_v1_Server_init_default           {false, {0, {0}}, false, 0, false, "", false, 0}
 #define grpc_lb_v1_Duration_init_zero            {false, 0, false, 0}
 #define grpc_lb_v1_LoadBalanceRequest_init_zero  {false, grpc_lb_v1_InitialLoadBalanceRequest_init_zero, false, grpc_lb_v1_ClientStats_init_zero}
 #define grpc_lb_v1_InitialLoadBalanceRequest_init_zero {false, ""}
@@ -122,7 +132,7 @@ typedef struct _grpc_lb_v1_LoadBalanceResponse {
 #define grpc_lb_v1_LoadBalanceResponse_init_zero {false, grpc_lb_v1_InitialLoadBalanceResponse_init_zero, false, grpc_lb_v1_ServerList_init_zero}
 #define grpc_lb_v1_InitialLoadBalanceResponse_init_zero {false, "", false, grpc_lb_v1_Duration_init_zero}
 #define grpc_lb_v1_ServerList_init_zero          {{{NULL}, NULL}, false, grpc_lb_v1_Duration_init_zero}
-#define grpc_lb_v1_Server_init_zero              {false, "", false, 0, false, "", false, 0}
+#define grpc_lb_v1_Server_init_zero              {false, {0, {0}}, false, 0, false, "", false, 0}
 
 /* Field tags (for use in manual encoding/decoding) */
 #define grpc_lb_v1_ClientStats_total_requests_tag 1
@@ -135,7 +145,7 @@ typedef struct _grpc_lb_v1_LoadBalanceResponse {
 #define grpc_lb_v1_Server_port_tag               2
 #define grpc_lb_v1_Server_load_balance_token_tag 3
 #define grpc_lb_v1_Server_drop_request_tag       4
-#define grpc_lb_v1_InitialLoadBalanceResponse_load_balancer_delegate_tag 2
+#define grpc_lb_v1_InitialLoadBalanceResponse_load_balancer_delegate_tag 1
 #define grpc_lb_v1_InitialLoadBalanceResponse_client_stats_report_interval_tag 3
 #define grpc_lb_v1_LoadBalanceRequest_initial_request_tag 1
 #define grpc_lb_v1_LoadBalanceRequest_client_stats_tag 2
@@ -161,7 +171,8 @@ extern const pb_field_t grpc_lb_v1_Server_fields[5];
 #define grpc_lb_v1_ClientStats_size              33
 #define grpc_lb_v1_LoadBalanceResponse_size      (98 + grpc_lb_v1_ServerList_size)
 #define grpc_lb_v1_InitialLoadBalanceResponse_size 90
-#define grpc_lb_v1_Server_size                   127
+/* grpc_lb_v1_ServerList_size depends on runtime parameters */
+#define grpc_lb_v1_Server_size                   98
 
 /* Message IDs (where set with "msgid" option) */
 #ifdef PB_MSGID
@@ -174,5 +185,6 @@ extern const pb_field_t grpc_lb_v1_Server_fields[5];
 #ifdef __cplusplus
 } /* extern "C" */
 #endif
+/* @@protoc_insertion_point(eof) */
 
 #endif
diff --git a/src/core/ext/lb_policy/pick_first/pick_first.c b/src/core/ext/lb_policy/pick_first/pick_first.c
index 9f3a6939e6..09df92dd99 100644
--- a/src/core/ext/lb_policy/pick_first/pick_first.c
+++ b/src/core/ext/lb_policy/pick_first/pick_first.c
@@ -199,10 +199,8 @@ static void pf_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
 }
 
 static int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
-                   grpc_polling_entity *pollent,
-                   grpc_metadata_batch *initial_metadata,
-                   uint32_t initial_metadata_flags,
-                   grpc_connected_subchannel **target,
+                   const grpc_lb_policy_pick_args *pick_args,
+                   grpc_connected_subchannel **target, void **user_data,
                    grpc_closure *on_complete) {
   pick_first_lb_policy *p = (pick_first_lb_policy *)pol;
   pending_pick *pp;
@@ -225,13 +223,13 @@ static int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     if (!p->started_picking) {
       start_picking(exec_ctx, p);
     }
-    grpc_polling_entity_add_to_pollset_set(exec_ctx, pollent,
+    grpc_polling_entity_add_to_pollset_set(exec_ctx, pick_args->pollent,
                                            p->base.interested_parties);
     pp = gpr_malloc(sizeof(*pp));
     pp->next = p->pending_picks;
-    pp->pollent = pollent;
+    pp->pollent = pick_args->pollent;
     pp->target = target;
-    pp->initial_metadata_flags = initial_metadata_flags;
+    pp->initial_metadata_flags = pick_args->initial_metadata_flags;
     pp->on_complete = on_complete;
     p->pending_picks = pp;
     gpr_mu_unlock(&p->mu);
@@ -462,6 +460,11 @@ static grpc_lb_policy *create_pick_first(grpc_exec_ctx *exec_ctx,
     /* Skip balancer addresses, since we only know how to handle backends. */
     if (args->addresses->addresses[i].is_balancer) continue;
 
+    if (args->addresses->addresses[i].user_data != NULL) {
+      gpr_log(GPR_ERROR,
+              "This LB policy doesn't support user data. It will be ignored");
+    }
+
     memset(&sc_args, 0, sizeof(grpc_subchannel_args));
     sc_args.addr =
         (struct sockaddr *)(&args->addresses->addresses[i].address.addr);
diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c
index fc1534887a..0feb0740a2 100644
--- a/src/core/ext/lb_policy/round_robin/round_robin.c
+++ b/src/core/ext/lb_policy/round_robin/round_robin.c
@@ -66,6 +66,7 @@
 #include "src/core/ext/client_config/lb_policy_registry.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/transport/connectivity_state.h"
+#include "src/core/lib/transport/static_metadata.h"
 
 typedef struct round_robin_lb_policy round_robin_lb_policy;
 
@@ -76,15 +77,32 @@ int grpc_lb_round_robin_trace = 0;
  * Once a pick is available, \a target is updated and \a on_complete called. */
 typedef struct pending_pick {
   struct pending_pick *next;
+
+  /* polling entity for the pick()'s async notification */
   grpc_polling_entity *pollent;
+
+  /* output argument where to store the pick()ed user_data. It'll be NULL if no
+   * such data is present or there's an error (the definite test for errors is
+   * \a target being NULL). */
+  void **user_data;
+
+  /* bitmask passed to pick() and used for selective cancelling. See
+   * grpc_lb_policy_cancel_picks() */
   uint32_t initial_metadata_flags;
+
+  /* output argument where to store the pick()ed connected subchannel, or NULL
+   * upon error. */
   grpc_connected_subchannel **target;
+
+  /* to be invoked once the pick() has completed (regardless of success) */
   grpc_closure *on_complete;
 } pending_pick;
 
 /** List of subchannels in a connectivity READY state */
 typedef struct ready_list {
   grpc_subchannel *subchannel;
+  /* references namesake entry in subchannel_data */
+  void *user_data;
   struct ready_list *next;
   struct ready_list *prev;
 } ready_list;
@@ -102,12 +120,17 @@ typedef struct {
   ready_list *ready_list_node;
   /** last observed connectivity */
   grpc_connectivity_state connectivity_state;
+  /** the subchannel's target user data */
+  void *user_data;
 } subchannel_data;
 
 struct round_robin_lb_policy {
   /** base policy: must be first */
   grpc_lb_policy base;
 
+  /** total number of addresses received at creation time */
+  size_t num_addresses;
+
   /** all our subchannels */
   size_t num_subchannels;
   subchannel_data **subchannels;
@@ -166,16 +189,19 @@ static void advance_last_picked_locked(round_robin_lb_policy *p) {
 
   if (grpc_lb_round_robin_trace) {
     gpr_log(GPR_DEBUG, "[READYLIST] ADVANCED LAST PICK. NOW AT NODE %p (SC %p)",
-            p->ready_list_last_pick, p->ready_list_last_pick->subchannel);
+            (void *)p->ready_list_last_pick,
+            (void *)p->ready_list_last_pick->subchannel);
   }
 }
 
 /** Prepends (relative to the root at p->ready_list) the connected subchannel \a
  * csc to the list of ready subchannels. */
 static ready_list *add_connected_sc_locked(round_robin_lb_policy *p,
-                                           grpc_subchannel *sc) {
+                                           subchannel_data *sd) {
   ready_list *new_elem = gpr_malloc(sizeof(ready_list));
-  new_elem->subchannel = sc;
+  memset(new_elem, 0, sizeof(ready_list));
+  new_elem->subchannel = sd->subchannel;
+  new_elem->user_data = sd->user_data;
   if (p->ready_list.prev == NULL) {
     /* first element */
     new_elem->next = &p->ready_list;
@@ -189,7 +215,8 @@ static ready_list *add_connected_sc_locked(round_robin_lb_policy *p,
     p->ready_list.prev = new_elem;
   }
   if (grpc_lb_round_robin_trace) {
-    gpr_log(GPR_DEBUG, "[READYLIST] ADDING NODE %p (SC %p)", new_elem, sc);
+    gpr_log(GPR_DEBUG, "[READYLIST] ADDING NODE %p (Conn. SC %p)",
+            (void *)new_elem, (void *)sd->subchannel);
   }
   return new_elem;
 }
@@ -216,8 +243,8 @@ static void remove_disconnected_sc_locked(round_robin_lb_policy *p,
   }
 
   if (grpc_lb_round_robin_trace) {
-    gpr_log(GPR_DEBUG, "[READYLIST] REMOVED NODE %p (SC %p)", node,
-            node->subchannel);
+    gpr_log(GPR_DEBUG, "[READYLIST] REMOVED NODE %p (SC %p)", (void *)node,
+            (void *)node->subchannel);
   }
 
   node->next = NULL;
@@ -229,9 +256,8 @@ static void remove_disconnected_sc_locked(round_robin_lb_policy *p,
 
 static void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   round_robin_lb_policy *p = (round_robin_lb_policy *)pol;
-  size_t i;
   ready_list *elem;
-  for (i = 0; i < p->num_subchannels; i++) {
+  for (size_t i = 0; i < p->num_subchannels; i++) {
     subchannel_data *sd = p->subchannels[i];
     GRPC_SUBCHANNEL_UNREF(exec_ctx, sd->subchannel, "round_robin");
     gpr_free(sd);
@@ -251,6 +277,7 @@ static void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
     gpr_free(elem);
     elem = tmp;
   }
+
   gpr_free(p);
 }
 
@@ -337,7 +364,7 @@ static void start_picking(grpc_exec_ctx *exec_ctx, round_robin_lb_policy *p) {
   p->started_picking = 1;
 
   if (grpc_lb_round_robin_trace) {
-    gpr_log(GPR_DEBUG, "LB_POLICY: p=%p num_subchannels=%" PRIuPTR, p,
+    gpr_log(GPR_DEBUG, "LB_POLICY: p=%p num_subchannels=%" PRIuPTR, (void *)p,
             p->num_subchannels);
   }
 
@@ -361,38 +388,43 @@ static void rr_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
 }
 
 static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
-                   grpc_polling_entity *pollent,
-                   grpc_metadata_batch *initial_metadata,
-                   uint32_t initial_metadata_flags,
-                   grpc_connected_subchannel **target,
+                   const grpc_lb_policy_pick_args *pick_args,
+                   grpc_connected_subchannel **target, void **user_data,
                    grpc_closure *on_complete) {
   round_robin_lb_policy *p = (round_robin_lb_policy *)pol;
   pending_pick *pp;
   ready_list *selected;
   gpr_mu_lock(&p->mu);
   if ((selected = peek_next_connected_locked(p))) {
+    /* readily available, report right away */
     gpr_mu_unlock(&p->mu);
     *target = grpc_subchannel_get_connected_subchannel(selected->subchannel);
+
+    if (user_data != NULL) {
+      *user_data = selected->user_data;
+    }
     if (grpc_lb_round_robin_trace) {
       gpr_log(GPR_DEBUG,
-              "[RR PICK] TARGET <-- CONNECTED SUBCHANNEL %p (NODE %p)", *target,
-              selected);
+              "[RR PICK] TARGET <-- CONNECTED SUBCHANNEL %p (NODE %p)",
+              (void *)*target, (void *)selected);
     }
     /* only advance the last picked pointer if the selection was used */
     advance_last_picked_locked(p);
     return 1;
   } else {
+    /* no pick currently available. Save for later in list of pending picks */
     if (!p->started_picking) {
       start_picking(exec_ctx, p);
     }
-    grpc_polling_entity_add_to_pollset_set(exec_ctx, pollent,
+    grpc_polling_entity_add_to_pollset_set(exec_ctx, pick_args->pollent,
                                            p->base.interested_parties);
     pp = gpr_malloc(sizeof(*pp));
     pp->next = p->pending_picks;
-    pp->pollent = pollent;
+    pp->pollent = pick_args->pollent;
     pp->target = target;
     pp->on_complete = on_complete;
-    pp->initial_metadata_flags = initial_metadata_flags;
+    pp->initial_metadata_flags = pick_args->initial_metadata_flags;
+    pp->user_data = user_data;
     p->pending_picks = pp;
     gpr_mu_unlock(&p->mu);
     return 0;
@@ -421,7 +453,7 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
                                     "connecting_ready");
         /* add the newly connected subchannel to the list of connected ones.
          * Note that it goes to the "end of the line". */
-        sd->ready_list_node = add_connected_sc_locked(p, sd->subchannel);
+        sd->ready_list_node = add_connected_sc_locked(p, sd);
         /* at this point we know there's at least one suitable subchannel. Go
          * ahead and pick one and notify the pending suitors in
          * p->pending_picks. This preemtively replicates rr_pick()'s actions. */
@@ -433,12 +465,16 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
         }
         while ((pp = p->pending_picks)) {
           p->pending_picks = pp->next;
+
           *pp->target =
               grpc_subchannel_get_connected_subchannel(selected->subchannel);
+          if (pp->user_data != NULL) {
+            *pp->user_data = selected->user_data;
+          }
           if (grpc_lb_round_robin_trace) {
             gpr_log(GPR_DEBUG,
                     "[RR CONN CHANGED] TARGET <-- SUBCHANNEL %p (NODE %p)",
-                    selected->subchannel, selected);
+                    (void *)selected->subchannel, (void *)selected);
           }
           grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent,
                                                    p->base.interested_parties);
@@ -582,6 +618,7 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx,
   round_robin_lb_policy *p = gpr_malloc(sizeof(*p));
   memset(p, 0, sizeof(*p));
 
+  p->num_addresses = num_addrs;
   p->subchannels = gpr_malloc(sizeof(*p->subchannels) * num_addrs);
   memset(p->subchannels, 0, sizeof(*p->subchannels) * num_addrs);
 
@@ -606,12 +643,14 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx,
       sd->policy = p;
       sd->index = subchannel_idx;
       sd->subchannel = subchannel;
+      sd->user_data = args->addresses->addresses[i].user_data;
       ++subchannel_idx;
       grpc_closure_init(&sd->connectivity_changed_closure,
                         rr_connectivity_changed, sd);
     }
   }
   if (subchannel_idx == 0) {
+    /* couldn't create any subchannel. Bail out */
     gpr_free(p->subchannels);
     gpr_free(p);
     return NULL;
diff --git a/src/core/ext/resolver/dns/native/dns_resolver.c b/src/core/ext/resolver/dns/native/dns_resolver.c
index 103d3effbd..1b85d2d634 100644
--- a/src/core/ext/resolver/dns/native/dns_resolver.c
+++ b/src/core/ext/resolver/dns/native/dns_resolver.c
@@ -167,11 +167,13 @@ static void dns_on_resolved(grpc_exec_ctx *exec_ctx, void *arg,
   GPR_ASSERT(r->resolving);
   r->resolving = 0;
   if (r->addresses != NULL) {
-    grpc_addresses *addresses = grpc_addresses_create(r->addresses->naddrs);
+    grpc_lb_addresses *addresses =
+        grpc_lb_addresses_create(r->addresses->naddrs);
     for (size_t i = 0; i < r->addresses->naddrs; ++i) {
-      grpc_addresses_set_address(addresses, i, &r->addresses->addrs[i].addr,
-                                 r->addresses->addrs[i].len,
-                                 false /* is_balancer */);
+      grpc_lb_addresses_set_address(
+          addresses, i, &r->addresses->addrs[i].addr,
+          r->addresses->addrs[i].len, false /* is_balancer */,
+          NULL /* balancer_name */, NULL /* user_data */);
     }
     grpc_resolved_addresses_destroy(r->addresses);
     result = grpc_resolver_result_create(addresses, r->lb_policy_name, NULL);
diff --git a/src/core/ext/resolver/sockaddr/sockaddr_resolver.c b/src/core/ext/resolver/sockaddr/sockaddr_resolver.c
index fb67f1a227..ced4fb024c 100644
--- a/src/core/ext/resolver/sockaddr/sockaddr_resolver.c
+++ b/src/core/ext/resolver/sockaddr/sockaddr_resolver.c
@@ -55,7 +55,7 @@ typedef struct {
   char *lb_policy_name;
 
   /** the addresses that we've 'resolved' */
-  grpc_addresses *addresses;
+  grpc_lb_addresses *addresses;
 
   /** mutex guarding the rest of the state */
   gpr_mu mu;
@@ -121,7 +121,8 @@ static void sockaddr_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
   if (r->next_completion != NULL && !r->published) {
     r->published = true;
     *r->target_result = grpc_resolver_result_create(
-        grpc_addresses_copy(r->addresses), r->lb_policy_name, NULL);
+        grpc_lb_addresses_copy(r->addresses, NULL /* user_data_copy */),
+        r->lb_policy_name, NULL);
     grpc_exec_ctx_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
     r->next_completion = NULL;
   }
@@ -130,7 +131,7 @@ static void sockaddr_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
 static void sockaddr_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *gr) {
   sockaddr_resolver *r = (sockaddr_resolver *)gr;
   gpr_mu_destroy(&r->mu);
-  grpc_addresses_destroy(r->addresses);
+  grpc_lb_addresses_destroy(r->addresses, NULL /* user_data_destroy */);
   gpr_free(r->lb_policy_name);
   gpr_free(r);
 }
@@ -204,7 +205,7 @@ static grpc_resolver *sockaddr_create(
   gpr_slice_buffer_init(&path_parts);
 
   gpr_slice_split(path_slice, ",", &path_parts);
-  r->addresses = grpc_addresses_create(path_parts.count);
+  r->addresses = grpc_lb_addresses_create(path_parts.count);
   for (size_t i = 0; i < r->addresses->num_addresses; i++) {
     grpc_uri ith_uri = *args->uri;
     char *part_str = gpr_dump_slice(path_parts.slices[i], GPR_DUMP_ASCII);
@@ -223,7 +224,7 @@ static grpc_resolver *sockaddr_create(
   gpr_slice_unref(path_slice);
   if (errors_found) {
     gpr_free(r->lb_policy_name);
-    grpc_addresses_destroy(r->addresses);
+    grpc_lb_addresses_destroy(r->addresses, NULL /* user_data_destroy */);
     gpr_free(r);
     return NULL;
   }
diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c
index 029c15014e..366690acf2 100644
--- a/src/core/ext/transport/cronet/transport/cronet_transport.c
+++ b/src/core/ext/transport/cronet/transport/cronet_transport.c
@@ -294,7 +294,7 @@ static void remove_from_storage(struct stream_obj *s,
 /*
   Cycle through ops and try to take next action. Break when either
   an action with callback is taken, or no action is possible.
-  This can be executed from the Cronet network thread via cronet callback
+  This can get executed from the Cronet network thread via cronet callback
   or on the application supplied thread via the perform_stream_op function.
 */
 static void execute_from_storage(stream_obj *s) {
@@ -329,6 +329,7 @@ static void execute_from_storage(stream_obj *s) {
 static void on_failed(cronet_bidirectional_stream *stream, int net_error) {
   CRONET_LOG(GPR_DEBUG, "on_failed(%p, %d)", stream, net_error);
   stream_obj *s = (stream_obj *)stream->annotation;
+  gpr_mu_lock(&s->mu);
   cronet_bidirectional_stream_destroy(s->cbs);
   s->state.state_callback_received[OP_FAILED] = true;
   s->cbs = NULL;
@@ -340,6 +341,7 @@ static void on_failed(cronet_bidirectional_stream *stream, int net_error) {
     gpr_free(s->state.ws.write_buffer);
     s->state.ws.write_buffer = NULL;
   }
+  gpr_mu_unlock(&s->mu);
   execute_from_storage(s);
 }
 
@@ -349,6 +351,7 @@ static void on_failed(cronet_bidirectional_stream *stream, int net_error) {
 static void on_canceled(cronet_bidirectional_stream *stream) {
   CRONET_LOG(GPR_DEBUG, "on_canceled(%p)", stream);
   stream_obj *s = (stream_obj *)stream->annotation;
+  gpr_mu_lock(&s->mu);
   cronet_bidirectional_stream_destroy(s->cbs);
   s->state.state_callback_received[OP_CANCELED] = true;
   s->cbs = NULL;
@@ -360,6 +363,7 @@ static void on_canceled(cronet_bidirectional_stream *stream) {
     gpr_free(s->state.ws.write_buffer);
     s->state.ws.write_buffer = NULL;
   }
+  gpr_mu_unlock(&s->mu);
   execute_from_storage(s);
 }
 
@@ -369,9 +373,11 @@ static void on_canceled(cronet_bidirectional_stream *stream) {
 static void on_succeeded(cronet_bidirectional_stream *stream) {
   CRONET_LOG(GPR_DEBUG, "on_succeeded(%p)", stream);
   stream_obj *s = (stream_obj *)stream->annotation;
+  gpr_mu_lock(&s->mu);
   cronet_bidirectional_stream_destroy(s->cbs);
   s->state.state_callback_received[OP_SUCCEEDED] = true;
   s->cbs = NULL;
+  gpr_mu_unlock(&s->mu);
   execute_from_storage(s);
 }
 
@@ -381,6 +387,7 @@ static void on_succeeded(cronet_bidirectional_stream *stream) {
 static void on_request_headers_sent(cronet_bidirectional_stream *stream) {
   CRONET_LOG(GPR_DEBUG, "W: on_request_headers_sent(%p)", stream);
   stream_obj *s = (stream_obj *)stream->annotation;
+  gpr_mu_lock(&s->mu);
   s->state.state_op_done[OP_SEND_INITIAL_METADATA] = true;
   s->state.state_callback_received[OP_SEND_INITIAL_METADATA] = true;
   /* Free the memory allocated for headers */
@@ -388,6 +395,7 @@ static void on_request_headers_sent(cronet_bidirectional_stream *stream) {
     gpr_free(s->header_array.headers);
     s->header_array.headers = NULL;
   }
+  gpr_mu_unlock(&s->mu);
   execute_from_storage(s);
 }
 
@@ -401,6 +409,7 @@ static void on_response_headers_received(
   CRONET_LOG(GPR_DEBUG, "R: on_response_headers_received(%p, %p, %s)", stream,
              headers, negotiated_protocol);
   stream_obj *s = (stream_obj *)stream->annotation;
+  gpr_mu_lock(&s->mu);
   memset(&s->state.rs.initial_metadata, 0,
          sizeof(s->state.rs.initial_metadata));
   grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.initial_metadata);
@@ -412,6 +421,7 @@ static void on_response_headers_received(
             grpc_mdstr_from_string(headers->headers[i].value)));
   }
   s->state.state_callback_received[OP_RECV_INITIAL_METADATA] = true;
+  gpr_mu_unlock(&s->mu);
   execute_from_storage(s);
 }
 
@@ -422,11 +432,13 @@ static void on_write_completed(cronet_bidirectional_stream *stream,
                                const char *data) {
   stream_obj *s = (stream_obj *)stream->annotation;
   CRONET_LOG(GPR_DEBUG, "W: on_write_completed(%p, %s)", stream, data);
+  gpr_mu_lock(&s->mu);
   if (s->state.ws.write_buffer) {
     gpr_free(s->state.ws.write_buffer);
     s->state.ws.write_buffer = NULL;
   }
   s->state.state_callback_received[OP_SEND_MESSAGE] = true;
+  gpr_mu_unlock(&s->mu);
   execute_from_storage(s);
 }
 
@@ -438,6 +450,7 @@ static void on_read_completed(cronet_bidirectional_stream *stream, char *data,
   stream_obj *s = (stream_obj *)stream->annotation;
   CRONET_LOG(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data,
              count);
+  gpr_mu_lock(&s->mu);
   s->state.state_callback_received[OP_RECV_MESSAGE] = true;
   if (count > 0) {
     s->state.rs.received_bytes += count;
@@ -448,11 +461,14 @@ static void on_read_completed(cronet_bidirectional_stream *stream, char *data,
       cronet_bidirectional_stream_read(
           s->cbs, s->state.rs.read_buffer + s->state.rs.received_bytes,
           s->state.rs.remaining_bytes);
+      gpr_mu_unlock(&s->mu);
     } else {
+      gpr_mu_unlock(&s->mu);
       execute_from_storage(s);
     }
   } else {
     s->state.rs.read_stream_closed = true;
+    gpr_mu_unlock(&s->mu);
     execute_from_storage(s);
   }
 }
@@ -466,6 +482,7 @@ static void on_response_trailers_received(
   CRONET_LOG(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream,
              trailers);
   stream_obj *s = (stream_obj *)stream->annotation;
+  gpr_mu_lock(&s->mu);
   memset(&s->state.rs.trailing_metadata, 0,
          sizeof(s->state.rs.trailing_metadata));
   s->state.rs.trailing_metadata_valid = false;
@@ -481,6 +498,7 @@ static void on_response_trailers_received(
     s->state.rs.trailing_metadata_valid = true;
   }
   s->state.state_callback_received[OP_RECV_TRAILING_METADATA] = true;
+  gpr_mu_unlock(&s->mu);
   execute_from_storage(s);
 }
 
@@ -757,14 +775,15 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
              op_can_be_run(stream_op, stream_state, &oas->state,
                            OP_RECV_INITIAL_METADATA)) {
     CRONET_LOG(GPR_DEBUG, "running: %p  OP_RECV_INITIAL_METADATA", oas);
-    if (!stream_state->state_op_done[OP_CANCEL_ERROR]) {
+    if (stream_state->state_op_done[OP_CANCEL_ERROR] ||
+        stream_state->state_callback_received[OP_FAILED]) {
+      grpc_exec_ctx_sched(exec_ctx, stream_op->recv_initial_metadata_ready,
+                          GRPC_ERROR_CANCELLED, NULL);
+    } else {
       grpc_chttp2_incoming_metadata_buffer_publish(
           &oas->s->state.rs.initial_metadata, stream_op->recv_initial_metadata);
       grpc_exec_ctx_sched(exec_ctx, stream_op->recv_initial_metadata_ready,
                           GRPC_ERROR_NONE, NULL);
-    } else {
-      grpc_exec_ctx_sched(exec_ctx, stream_op->recv_initial_metadata_ready,
-                          GRPC_ERROR_CANCELLED, NULL);
     }
     stream_state->state_op_done[OP_RECV_INITIAL_METADATA] = true;
     result = ACTION_TAKEN_NO_CALLBACK;
@@ -772,32 +791,40 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
              op_can_be_run(stream_op, stream_state, &oas->state,
                            OP_SEND_MESSAGE)) {
     CRONET_LOG(GPR_DEBUG, "running: %p  OP_SEND_MESSAGE", oas);
-    gpr_slice_buffer write_slice_buffer;
-    gpr_slice slice;
-    gpr_slice_buffer_init(&write_slice_buffer);
-    grpc_byte_stream_next(NULL, stream_op->send_message, &slice,
-                          stream_op->send_message->length, NULL);
-    /* Check that compression flag is OFF. We don't support compression yet. */
-    if (stream_op->send_message->flags != 0) {
-      gpr_log(GPR_ERROR, "Compression is not supported");
-      GPR_ASSERT(stream_op->send_message->flags == 0);
-    }
-    gpr_slice_buffer_add(&write_slice_buffer, slice);
-    if (write_slice_buffer.count != 1) {
-      /* Empty request not handled yet */
-      gpr_log(GPR_ERROR, "Empty request is not supported");
-      GPR_ASSERT(write_slice_buffer.count == 1);
-    }
-    if (write_slice_buffer.count > 0) {
-      size_t write_buffer_size;
-      create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer,
-                        &write_buffer_size);
-      CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p, %p)",
-                 s->cbs, stream_state->ws.write_buffer);
-      stream_state->state_callback_received[OP_SEND_MESSAGE] = false;
-      cronet_bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer,
-                                        (int)write_buffer_size, false);
-      result = ACTION_TAKEN_WITH_CALLBACK;
+    if (stream_state->state_callback_received[OP_FAILED]) {
+      result = NO_ACTION_POSSIBLE;
+      CRONET_LOG(GPR_DEBUG, "Stream is either cancelled or failed.");
+    } else {
+      gpr_slice_buffer write_slice_buffer;
+      gpr_slice slice;
+      gpr_slice_buffer_init(&write_slice_buffer);
+      grpc_byte_stream_next(NULL, stream_op->send_message, &slice,
+                            stream_op->send_message->length, NULL);
+      /* Check that compression flag is OFF. We don't support compression yet.
+       */
+      if (stream_op->send_message->flags != 0) {
+        gpr_log(GPR_ERROR, "Compression is not supported");
+        GPR_ASSERT(stream_op->send_message->flags == 0);
+      }
+      gpr_slice_buffer_add(&write_slice_buffer, slice);
+      if (write_slice_buffer.count != 1) {
+        /* Empty request not handled yet */
+        gpr_log(GPR_ERROR, "Empty request is not supported");
+        GPR_ASSERT(write_slice_buffer.count == 1);
+      }
+      if (write_slice_buffer.count > 0) {
+        size_t write_buffer_size;
+        create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer,
+                          &write_buffer_size);
+        CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p, %p)",
+                   s->cbs, stream_state->ws.write_buffer);
+        stream_state->state_callback_received[OP_SEND_MESSAGE] = false;
+        cronet_bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer,
+                                          (int)write_buffer_size, false);
+        result = ACTION_TAKEN_WITH_CALLBACK;
+      } else {
+        result = NO_ACTION_POSSIBLE;
+      }
     }
     stream_state->state_op_done[OP_SEND_MESSAGE] = true;
     oas->state.state_op_done[OP_SEND_MESSAGE] = true;
@@ -805,7 +832,9 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
              op_can_be_run(stream_op, stream_state, &oas->state,
                            OP_RECV_MESSAGE)) {
     CRONET_LOG(GPR_DEBUG, "running: %p  OP_RECV_MESSAGE", oas);
-    if (stream_state->state_op_done[OP_CANCEL_ERROR]) {
+    if (stream_state->state_op_done[OP_CANCEL_ERROR] ||
+        stream_state->state_callback_received[OP_FAILED]) {
+      CRONET_LOG(GPR_DEBUG, "Stream is either cancelled or failed.");
       grpc_exec_ctx_sched(exec_ctx, stream_op->recv_message_ready,
                           GRPC_ERROR_CANCELLED, NULL);
       stream_state->state_op_done[OP_RECV_MESSAGE] = true;
@@ -861,8 +890,10 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
             true; /* Indicates that at least one read request has been made */
         cronet_bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer,
                                          stream_state->rs.remaining_bytes);
+        result = ACTION_TAKEN_WITH_CALLBACK;
+      } else {
+        result = NO_ACTION_POSSIBLE;
       }
-      result = ACTION_TAKEN_WITH_CALLBACK;
     } else if (stream_state->rs.remaining_bytes == 0) {
       CRONET_LOG(GPR_DEBUG, "read operation complete");
       gpr_slice read_data_slice =
@@ -903,11 +934,17 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
              op_can_be_run(stream_op, stream_state, &oas->state,
                            OP_SEND_TRAILING_METADATA)) {
     CRONET_LOG(GPR_DEBUG, "running: %p  OP_SEND_TRAILING_METADATA", oas);
-    CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p, 0)", s->cbs);
-    stream_state->state_callback_received[OP_SEND_MESSAGE] = false;
-    cronet_bidirectional_stream_write(s->cbs, "", 0, true);
+    if (stream_state->state_callback_received[OP_FAILED]) {
+      result = NO_ACTION_POSSIBLE;
+      CRONET_LOG(GPR_DEBUG, "Stream is either cancelled or failed.");
+    } else {
+      CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p, 0)",
+                 s->cbs);
+      stream_state->state_callback_received[OP_SEND_MESSAGE] = false;
+      cronet_bidirectional_stream_write(s->cbs, "", 0, true);
+      result = ACTION_TAKEN_WITH_CALLBACK;
+    }
     stream_state->state_op_done[OP_SEND_TRAILING_METADATA] = true;
-    result = ACTION_TAKEN_WITH_CALLBACK;
   } else if (stream_op->cancel_error &&
              op_can_be_run(stream_op, stream_state, &oas->state,
                            OP_CANCEL_ERROR)) {
diff --git a/src/core/lib/channel/handshaker.c b/src/core/lib/channel/handshaker.c
index c0979f5e80..8f9fb17a31 100644
--- a/src/core/lib/channel/handshaker.c
+++ b/src/core/lib/channel/handshaker.c
@@ -33,8 +33,8 @@
 
 #include <string.h>
 
-#include <grpc/impl/codegen/alloc.h>
-#include <grpc/impl/codegen/log.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
 
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/handshaker.h"
diff --git a/src/core/lib/channel/handshaker.h b/src/core/lib/channel/handshaker.h
index b276f6028c..d574b46242 100644
--- a/src/core/lib/channel/handshaker.h
+++ b/src/core/lib/channel/handshaker.h
@@ -35,8 +35,6 @@
 #define GRPC_CORE_LIB_CHANNEL_HANDSHAKER_H
 
 #include <grpc/impl/codegen/grpc_types.h>
-#include <grpc/impl/codegen/time.h>
-#include <grpc/support/slice_buffer.h>
 
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/endpoint.h"
diff --git a/src/core/lib/security/credentials/jwt/json_token.c b/src/core/lib/security/credentials/jwt/json_token.c
index 354c13133e..192a5f47ed 100644
--- a/src/core/lib/security/credentials/jwt/json_token.c
+++ b/src/core/lib/security/credentials/jwt/json_token.c
@@ -38,6 +38,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
+#include <grpc/support/time.h>
 
 #include "src/core/lib/security/util/b64.h"
 #include "src/core/lib/security/util/json_util.h"
diff --git a/src/core/lib/support/time_posix.c b/src/core/lib/support/time_posix.c
index 11542072fe..a69c501e9f 100644
--- a/src/core/lib/support/time_posix.c
+++ b/src/core/lib/support/time_posix.c
@@ -127,7 +127,7 @@ static gpr_timespec now_impl(gpr_clock_type clock) {
       now.tv_nsec = now_tv.tv_usec * 1000;
       break;
     case GPR_CLOCK_MONOTONIC:
-      now_dbl = (mach_absolute_time() - g_time_start) * g_time_scale;
+      now_dbl = ((double)(mach_absolute_time() - g_time_start)) * g_time_scale;
       now.tv_sec = (int64_t)(now_dbl * 1e-9);
       now.tv_nsec = (int32_t)(now_dbl - ((double)now.tv_sec) * 1e9);
       break;
diff --git a/src/core/lib/transport/metadata.c b/src/core/lib/transport/metadata.c
index 0677f29766..4b40c275ad 100644
--- a/src/core/lib/transport/metadata.c
+++ b/src/core/lib/transport/metadata.c
@@ -278,7 +278,7 @@ static void ref_md_locked(mdtab_shard *shard,
                           internal_metadata *md DEBUG_ARGS) {
 #ifdef GRPC_METADATA_REFCOUNT_DEBUG
   gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-          "ELM   REF:%p:%d->%d: '%s' = '%s'", md,
+          "ELM   REF:%p:%zu->%zu: '%s' = '%s'", (void *)md,
           gpr_atm_no_barrier_load(&md->refcnt),
           gpr_atm_no_barrier_load(&md->refcnt) + 1,
           grpc_mdstr_as_c_string((grpc_mdstr *)md->key),
@@ -566,7 +566,7 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdstr *mkey,
   shard->elems[idx] = md;
   gpr_mu_init(&md->mu_user_data);
 #ifdef GRPC_METADATA_REFCOUNT_DEBUG
-  gpr_log(GPR_DEBUG, "ELM   NEW:%p:%d: '%s' = '%s'", md,
+  gpr_log(GPR_DEBUG, "ELM   NEW:%p:%zu: '%s' = '%s'", (void *)md,
           gpr_atm_no_barrier_load(&md->refcnt),
           grpc_mdstr_as_c_string((grpc_mdstr *)md->key),
           grpc_mdstr_as_c_string((grpc_mdstr *)md->value));
@@ -639,7 +639,7 @@ grpc_mdelem *grpc_mdelem_ref(grpc_mdelem *gmd DEBUG_ARGS) {
   if (is_mdelem_static(gmd)) return gmd;
 #ifdef GRPC_METADATA_REFCOUNT_DEBUG
   gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-          "ELM   REF:%p:%d->%d: '%s' = '%s'", md,
+          "ELM   REF:%p:%zu->%zu: '%s' = '%s'", (void *)md,
           gpr_atm_no_barrier_load(&md->refcnt),
           gpr_atm_no_barrier_load(&md->refcnt) + 1,
           grpc_mdstr_as_c_string((grpc_mdstr *)md->key),
@@ -649,7 +649,7 @@ grpc_mdelem *grpc_mdelem_ref(grpc_mdelem *gmd DEBUG_ARGS) {
      this function - meaning that no adjustment to mdtab_free is necessary,
      simplifying the logic here to be just an atomic increment */
   /* use C assert to have this removed in opt builds */
-  assert(gpr_atm_no_barrier_load(&md->refcnt) >= 1);
+  GPR_ASSERT(gpr_atm_no_barrier_load(&md->refcnt) >= 1);
   gpr_atm_no_barrier_fetch_add(&md->refcnt, 1);
   return gmd;
 }
@@ -660,14 +660,16 @@ void grpc_mdelem_unref(grpc_mdelem *gmd DEBUG_ARGS) {
   if (is_mdelem_static(gmd)) return;
 #ifdef GRPC_METADATA_REFCOUNT_DEBUG
   gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
-          "ELM UNREF:%p:%d->%d: '%s' = '%s'", md,
+          "ELM UNREF:%p:%zu->%zu: '%s' = '%s'", (void *)md,
           gpr_atm_no_barrier_load(&md->refcnt),
           gpr_atm_no_barrier_load(&md->refcnt) - 1,
           grpc_mdstr_as_c_string((grpc_mdstr *)md->key),
           grpc_mdstr_as_c_string((grpc_mdstr *)md->value));
 #endif
   uint32_t hash = GRPC_MDSTR_KV_HASH(md->key->hash, md->value->hash);
-  if (1 == gpr_atm_full_fetch_add(&md->refcnt, -1)) {
+  const gpr_atm prev_refcount = gpr_atm_full_fetch_add(&md->refcnt, -1);
+  GPR_ASSERT(prev_refcount >= 1);
+  if (1 == prev_refcount) {
     /* once the refcount hits zero, some other thread can come along and
        free md at any time: it's unsafe from this point on to access it */
     mdtab_shard *shard =
@@ -676,10 +678,12 @@ void grpc_mdelem_unref(grpc_mdelem *gmd DEBUG_ARGS) {
   }
 }
 
-const char *grpc_mdstr_as_c_string(grpc_mdstr *s) {
+const char *grpc_mdstr_as_c_string(const grpc_mdstr *s) {
   return (const char *)GPR_SLICE_START_PTR(s->slice);
 }
 
+size_t grpc_mdstr_length(const grpc_mdstr *s) { return GRPC_MDSTR_LENGTH(s); }
+
 grpc_mdstr *grpc_mdstr_ref(grpc_mdstr *gs DEBUG_ARGS) {
   internal_string *s = (internal_string *)gs;
   if (is_mdstr_static(gs)) return gs;
diff --git a/src/core/lib/transport/metadata.h b/src/core/lib/transport/metadata.h
index 2b0921c8d7..71eff0acf2 100644
--- a/src/core/lib/transport/metadata.h
+++ b/src/core/lib/transport/metadata.h
@@ -147,7 +147,7 @@ void grpc_mdelem_unref(grpc_mdelem *md);
 
 /* Recover a char* from a grpc_mdstr. The returned string is null terminated.
    Does not promise that the returned string has no embedded nulls however. */
-const char *grpc_mdstr_as_c_string(grpc_mdstr *s);
+const char *grpc_mdstr_as_c_string(const grpc_mdstr *s);
 
 #define GRPC_MDSTR_LENGTH(s) (GPR_SLICE_LENGTH(s->slice))
 
diff --git a/src/cpp/client/client_context.cc b/src/cpp/client/client_context.cc
index 74bb7b0d1c..5b6aaa777b 100644
--- a/src/cpp/client/client_context.cc
+++ b/src/cpp/client/client_context.cc
@@ -36,6 +36,7 @@
 #include <grpc/compression.h>
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
 #include <grpc++/security/credentials.h>
diff --git a/src/cpp/common/channel_filter.h b/src/cpp/common/channel_filter.h
index ec2af63afd..6f5af3dec3 100644
--- a/src/cpp/common/channel_filter.h
+++ b/src/cpp/common/channel_filter.h
@@ -36,7 +36,7 @@
 
 #include <grpc++/impl/codegen/config.h>
 #include <grpc/grpc.h>
-#include <grpc/impl/codegen/alloc.h>
+#include <grpc/support/alloc.h>
 
 #include <functional>
 #include <vector>
diff --git a/src/cpp/common/core_codegen.cc b/src/cpp/common/core_codegen.cc
index 3d6780bcb8..ce02202976 100644
--- a/src/cpp/common/core_codegen.cc
+++ b/src/cpp/common/core_codegen.cc
@@ -39,15 +39,19 @@
 #include <grpc/byte_buffer.h>
 #include <grpc/byte_buffer_reader.h>
 #include <grpc/grpc.h>
-#include <grpc/impl/codegen/alloc.h>
-#include <grpc/impl/codegen/byte_buffer.h>
-#include <grpc/impl/codegen/log.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
 #include <grpc/support/port_platform.h>
 #include <grpc/support/slice.h>
 #include <grpc/support/slice_buffer.h>
+#include <grpc/support/sync.h>
 
 #include "src/core/lib/profiling/timers.h"
 
+extern "C" {
+struct grpc_byte_buffer;
+}
+
 namespace grpc {
 
 grpc_completion_queue* CoreCodegen::grpc_completion_queue_create(
@@ -70,6 +74,19 @@ void* CoreCodegen::gpr_malloc(size_t size) { return ::gpr_malloc(size); }
 
 void CoreCodegen::gpr_free(void* p) { return ::gpr_free(p); }
 
+void CoreCodegen::gpr_mu_init(gpr_mu* mu) { ::gpr_mu_init(mu); };
+void CoreCodegen::gpr_mu_destroy(gpr_mu* mu) { ::gpr_mu_destroy(mu); }
+void CoreCodegen::gpr_mu_lock(gpr_mu* mu) { ::gpr_mu_lock(mu); }
+void CoreCodegen::gpr_mu_unlock(gpr_mu* mu) { ::gpr_mu_unlock(mu); }
+void CoreCodegen::gpr_cv_init(gpr_cv* cv) { ::gpr_cv_init(cv); }
+void CoreCodegen::gpr_cv_destroy(gpr_cv* cv) { ::gpr_cv_destroy(cv); }
+int CoreCodegen::gpr_cv_wait(gpr_cv* cv, gpr_mu* mu,
+                             gpr_timespec abs_deadline) {
+  return ::gpr_cv_wait(cv, mu, abs_deadline);
+}
+void CoreCodegen::gpr_cv_signal(gpr_cv* cv) { ::gpr_cv_signal(cv); }
+void CoreCodegen::gpr_cv_broadcast(gpr_cv* cv) { ::gpr_cv_broadcast(cv); }
+
 void CoreCodegen::grpc_byte_buffer_destroy(grpc_byte_buffer* bb) {
   ::grpc_byte_buffer_destroy(bb);
 }
@@ -128,6 +145,10 @@ gpr_timespec CoreCodegen::gpr_inf_future(gpr_clock_type type) {
   return ::gpr_inf_future(type);
 }
 
+gpr_timespec CoreCodegen::gpr_time_0(gpr_clock_type type) {
+  return ::gpr_time_0(type);
+}
+
 void CoreCodegen::assert_fail(const char* failed_assertion) {
   gpr_log(GPR_ERROR, "assertion failed: %s", failed_assertion);
   abort();
diff --git a/src/csharp/Grpc.Tools.nuspec b/src/csharp/Grpc.Tools.nuspec
index 31d1bed647..0c937ab9cb 100644
--- a/src/csharp/Grpc.Tools.nuspec
+++ b/src/csharp/Grpc.Tools.nuspec
@@ -16,17 +16,18 @@
     <tags>gRPC RPC Protocol HTTP/2</tags>
   </metadata>
   <files>
-    <file src="protoc_plugins\windows_x86\protoc.exe" target="tools\windows_x86\protoc.exe" />
-    <file src="protoc_plugins\windows_x86\grpc_csharp_plugin.exe" target="tools\windows_x86\grpc_csharp_plugin.exe" />
-    <file src="protoc_plugins\windows_x64\protoc.exe" target="tools\windows_x64\protoc.exe" />
-    <file src="protoc_plugins\windows_x64\grpc_csharp_plugin.exe" target="tools\windows_x64\grpc_csharp_plugin.exe" />
-    <file src="protoc_plugins\linux_x86\protoc" target="tools\linux_x86\protoc" />
-    <file src="protoc_plugins\linux_x86\grpc_csharp_plugin" target="tools\linux_x86\grpc_csharp_plugin" />
-    <file src="protoc_plugins\linux_x64\protoc" target="tools\linux_x64\protoc" />
-    <file src="protoc_plugins\linux_x64\grpc_csharp_plugin" target="tools\linux_x64\grpc_csharp_plugin" />
-    <file src="protoc_plugins\macosx_x86\protoc" target="tools\macosx_x86\protoc" />
-    <file src="protoc_plugins\macosx_x86\grpc_csharp_plugin" target="tools\macosx_x86\grpc_csharp_plugin" />
-    <file src="protoc_plugins\macosx_x64\protoc" target="tools\macosx_x64\protoc" />
-    <file src="protoc_plugins\macosx_x64\grpc_csharp_plugin" target="tools\macosx_x64\grpc_csharp_plugin" />
+    <!-- forward slashes in src path enable building on Linux -->
+    <file src="protoc_plugins/windows_x86/protoc.exe" target="tools\windows_x86\protoc.exe" />
+    <file src="protoc_plugins/windows_x86/grpc_csharp_plugin.exe" target="tools\windows_x86\grpc_csharp_plugin.exe" />
+    <file src="protoc_plugins/windows_x64/protoc.exe" target="tools\windows_x64\protoc.exe" />
+    <file src="protoc_plugins/windows_x64/grpc_csharp_plugin.exe" target="tools\windows_x64\grpc_csharp_plugin.exe" />
+    <file src="protoc_plugins/linux_x86/protoc" target="tools\linux_x86\protoc" />
+    <file src="protoc_plugins/linux_x86/grpc_csharp_plugin" target="tools\linux_x86\grpc_csharp_plugin" />
+    <file src="protoc_plugins/linux_x64/protoc" target="tools\linux_x64\protoc" />
+    <file src="protoc_plugins/linux_x64/grpc_csharp_plugin" target="tools\linux_x64\grpc_csharp_plugin" />
+    <file src="protoc_plugins/macosx_x86/protoc" target="tools\macosx_x86\protoc" />
+    <file src="protoc_plugins/macosx_x86/grpc_csharp_plugin" target="tools\macosx_x86\grpc_csharp_plugin" />
+    <file src="protoc_plugins/macosx_x64/protoc" target="tools\macosx_x64\protoc" />
+    <file src="protoc_plugins/macosx_x64/grpc_csharp_plugin" target="tools\macosx_x64\grpc_csharp_plugin" />
   </files>
 </package>
diff --git a/src/csharp/build_packages.bat b/src/csharp/build_packages.bat
index b92189c840..c0036df13f 100644
--- a/src/csharp/build_packages.bat
+++ b/src/csharp/build_packages.bat
@@ -29,6 +29,10 @@
 
 @rem Builds gRPC NuGet packages
 
+@rem This way of building nuget packages is now obsolete. C# nuget packages
+@rem with CoreCLR support are now being built using the dotnet cli
+@rem in build_packages_dotnetcli.sh
+
 @rem Current package versions
 set VERSION=1.1.0-dev
 set PROTOBUF_VERSION=3.0.0
@@ -77,8 +81,8 @@ endlocal
 xcopy /Y /I *.nupkg ..\..\artifacts\
 
 @rem create a zipfile with the artifacts as well
-powershell -Command "Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('..\..\artifacts', 'csharp_nugets.zip');"
-xcopy /Y /I csharp_nugets.zip ..\..\artifacts\
+powershell -Command "Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('..\..\artifacts', 'csharp_nugets_obsolete.zip');"
+xcopy /Y /I csharp_nugets_obsolete.zip ..\..\artifacts\
 
 goto :EOF
 
diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh
index 2807278de6..3c127f320b 100755
--- a/src/csharp/build_packages_dotnetcli.sh
+++ b/src/csharp/build_packages_dotnetcli.sh
@@ -34,9 +34,6 @@ cd $(dirname $0)
 
 mkdir -p ../../artifacts/
 
-# IMPORTANT: NuGet packages generated by dotnet CLI are considered experimental.
-# The official nugets are generated by src/csharp/build_packages.bat
-
 mkdir -p nativelibs/windows_x86 nativelibs/windows_x64 \
     nativelibs/linux_x86 nativelibs/linux_x64 \
     nativelibs/macosx_x86 nativelibs/macosx_x64
diff --git a/src/proto/grpc/lb/v1/load_balancer.options b/src/proto/grpc/lb/v1/load_balancer.options
index d90366996e..a9398d5f47 100644
--- a/src/proto/grpc/lb/v1/load_balancer.options
+++ b/src/proto/grpc/lb/v1/load_balancer.options
@@ -1,6 +1,6 @@
 grpc.lb.v1.InitialLoadBalanceRequest.name max_size:128
 grpc.lb.v1.InitialLoadBalanceResponse.client_config max_size:64
 grpc.lb.v1.InitialLoadBalanceResponse.load_balancer_delegate max_size:64
-grpc.lb.v1.Server.ip_address max_size:46
-grpc.lb.v1.Server.load_balance_token max_size:64
+grpc.lb.v1.Server.ip_address max_size:16
+grpc.lb.v1.Server.load_balance_token max_size:65
 load_balancer.proto no_unions:true
diff --git a/src/proto/grpc/lb/v1/load_balancer.proto b/src/proto/grpc/lb/v1/load_balancer.proto
index 1bcad0b1d4..b4a33f3338 100644
--- a/src/proto/grpc/lb/v1/load_balancer.proto
+++ b/src/proto/grpc/lb/v1/load_balancer.proto
@@ -32,7 +32,6 @@ syntax = "proto3";
 package grpc.lb.v1;
 
 message Duration {
-
   // Signed seconds of the span of time. Must be from -315,576,000,000
   // to +315,576,000,000 inclusive.
   int64 seconds = 1;
@@ -93,16 +92,11 @@ message LoadBalanceResponse {
 }
 
 message InitialLoadBalanceResponse {
-  oneof initial_response_type {
-    // TODO(zhangkun83): ClientConfig not yet defined
-    //ClientConfig client_config = 1;
-
-    // This is an application layer redirect that indicates the client should
-    // use the specified server for load balancing. When this field is set in
-    // the response, the client should open a separate connection to the
-    // load_balancer_delegate and call the BalanceLoad method.
-    string load_balancer_delegate = 2;
-  }
+  // This is an application layer redirect that indicates the client should use
+  // the specified server for load balancing. When this field is non-empty in
+  // the response, the client should open a separate connection to the
+  // load_balancer_delegate and call the BalanceLoad method.
+  string load_balancer_delegate = 1;
 
   // This interval defines how often the client should send the client stats
   // to the load balancer. Stats should only be reported when the duration is
@@ -125,14 +119,17 @@ message ServerList {
 }
 
 message Server {
-  // A resolved address and port for the server. The IP address string may
+  // A resolved address for the server, serialized in network-byte-order. It may
   // either be an IPv4 or IPv6 address.
-  string ip_address = 1;
+  bytes ip_address = 1;
+
+  // A resolved port number for the server.
   int32 port = 2;
 
-  // An opaque token that is passed from the client to the server in metadata.
-  // The server may expect this token to indicate that the request from the
-  // client was load balanced.
+  // An opaque but printable token given to the frontend for each pick. All
+  // frontend requests for that pick must include the token in its initial
+  // metadata. The token is used by the backend to verify the request and to
+  // allow the backend to report load to the gRPC LB system.
   string load_balance_token = 3;
 
   // Indicates whether this particular request should be dropped by the client
diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py
index d36ac23305..ea3b6f3391 100644
--- a/src/python/grpcio/commands.py
+++ b/src/python/grpcio/commands.py
@@ -53,6 +53,7 @@ PYTHON_STEM = os.path.dirname(os.path.abspath(__file__))
 GRPC_STEM = os.path.abspath(PYTHON_STEM + '../../../../')
 PROTO_STEM = os.path.join(GRPC_STEM, 'src', 'proto')
 PROTO_GEN_STEM = os.path.join(GRPC_STEM, 'src', 'python', 'gens')
+CYTHON_STEM = os.path.join(PYTHON_STEM, 'grpc', '_cython')
 
 CONF_PY_ADDENDUM = """
 extensions.append('sphinx.ext.napoleon')
@@ -244,7 +245,7 @@ def try_cythonize(extensions, linetracing=False, mandatory=True):
     extensions,
     include_path=[
       include_dir for extension in extensions for include_dir in extension.include_dirs
-    ],
+    ] + [CYTHON_STEM],
     compiler_directives=cython_compiler_directives
   )
 
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/security.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/security.pyx.pxi
index 23cee7bd6e..20fc1c5fce 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/security.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/security.pyx.pxi
@@ -35,7 +35,7 @@ import pkg_resources
 cdef grpc_ssl_roots_override_result ssl_roots_override_callback(
     char **pem_root_certs) with gil:
   temporary_pem_root_certs = pkg_resources.resource_string(
-      'grpc._cython', '_credentials/roots.pem')
+      __name__.rstrip('.cygrpc'), '_credentials/roots.pem')
   pem_root_certs[0] = <char *>gpr_malloc(len(temporary_pem_root_certs) + 1)
   memcpy(
       pem_root_certs[0], <char *>temporary_pem_root_certs,
diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pxd b/src/python/grpcio/grpc/_cython/cygrpc.pxd
index 9779534e38..26454634a1 100644
--- a/src/python/grpcio/grpc/_cython/cygrpc.pxd
+++ b/src/python/grpcio/grpc/_cython/cygrpc.pxd
@@ -27,12 +27,12 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-include "grpc/_cython/_cygrpc/grpc.pxi"
+include "_cygrpc/grpc.pxi"
 
-include "grpc/_cython/_cygrpc/call.pxd.pxi"
-include "grpc/_cython/_cygrpc/channel.pxd.pxi"
-include "grpc/_cython/_cygrpc/credentials.pxd.pxi"
-include "grpc/_cython/_cygrpc/completion_queue.pxd.pxi"
-include "grpc/_cython/_cygrpc/records.pxd.pxi"
-include "grpc/_cython/_cygrpc/security.pxd.pxi"
-include "grpc/_cython/_cygrpc/server.pxd.pxi"
+include "_cygrpc/call.pxd.pxi"
+include "_cygrpc/channel.pxd.pxi"
+include "_cygrpc/credentials.pxd.pxi"
+include "_cygrpc/completion_queue.pxd.pxi"
+include "_cygrpc/records.pxd.pxi"
+include "_cygrpc/security.pxd.pxi"
+include "_cygrpc/server.pxd.pxi"
diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pyx b/src/python/grpcio/grpc/_cython/cygrpc.pyx
index 08089994a9..e1bd046a1a 100644
--- a/src/python/grpcio/grpc/_cython/cygrpc.pyx
+++ b/src/python/grpcio/grpc/_cython/cygrpc.pyx
@@ -35,14 +35,14 @@ import sys
 
 # TODO(atash): figure out why the coverage tool gets confused about the Cython
 # coverage plugin when the following files don't have a '.pxi' suffix.
-include "grpc/_cython/_cygrpc/grpc_string.pyx.pxi"
-include "grpc/_cython/_cygrpc/call.pyx.pxi"
-include "grpc/_cython/_cygrpc/channel.pyx.pxi"
-include "grpc/_cython/_cygrpc/credentials.pyx.pxi"
-include "grpc/_cython/_cygrpc/completion_queue.pyx.pxi"
-include "grpc/_cython/_cygrpc/records.pyx.pxi"
-include "grpc/_cython/_cygrpc/security.pyx.pxi"
-include "grpc/_cython/_cygrpc/server.pyx.pxi"
+include "_cygrpc/grpc_string.pyx.pxi"
+include "_cygrpc/call.pyx.pxi"
+include "_cygrpc/channel.pyx.pxi"
+include "_cygrpc/credentials.pyx.pxi"
+include "_cygrpc/completion_queue.pyx.pxi"
+include "_cygrpc/records.pyx.pxi"
+include "_cygrpc/security.pyx.pxi"
+include "_cygrpc/server.pyx.pxi"
 
 #
 # initialize gRPC
diff --git a/src/ruby/ext/grpc/rb_compression_options.c b/src/ruby/ext/grpc/rb_compression_options.c
index 0a3a215b1c..c5668fdab4 100644
--- a/src/ruby/ext/grpc/rb_compression_options.c
+++ b/src/ruby/ext/grpc/rb_compression_options.c
@@ -38,7 +38,7 @@
 
 #include <grpc/compression.h>
 #include <grpc/grpc.h>
-#include <grpc/impl/codegen/alloc.h>
+#include <grpc/support/alloc.h>
 #include <grpc/impl/codegen/compression_types.h>
 #include <grpc/impl/codegen/grpc_types.h>
 #include <string.h>
diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c
index 9caaf7b783..a6cad0db1a 100644
--- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c
+++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c
@@ -37,6 +37,16 @@
 
 #include "rb_grpc_imports.generated.h"
 
+grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import;
+grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import;
+grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import;
+grpc_byte_buffer_length_type grpc_byte_buffer_length_import;
+grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import;
+grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import;
+grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import;
+grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import;
+grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import;
+grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import;
 census_initialize_type census_initialize_import;
 census_shutdown_type census_shutdown_import;
 census_supported_type census_supported_import;
@@ -164,21 +174,48 @@ gpr_malloc_aligned_type gpr_malloc_aligned_import;
 gpr_free_aligned_type gpr_free_aligned_import;
 gpr_set_allocation_functions_type gpr_set_allocation_functions_import;
 gpr_get_allocation_functions_type gpr_get_allocation_functions_import;
-grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import;
-grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import;
-grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import;
-grpc_byte_buffer_length_type grpc_byte_buffer_length_import;
-grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import;
-grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import;
-grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import;
-grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import;
-grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import;
-grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import;
+gpr_avl_create_type gpr_avl_create_import;
+gpr_avl_ref_type gpr_avl_ref_import;
+gpr_avl_unref_type gpr_avl_unref_import;
+gpr_avl_add_type gpr_avl_add_import;
+gpr_avl_remove_type gpr_avl_remove_import;
+gpr_avl_get_type gpr_avl_get_import;
+gpr_avl_maybe_get_type gpr_avl_maybe_get_import;
+gpr_avl_is_empty_type gpr_avl_is_empty_import;
+gpr_cmdline_create_type gpr_cmdline_create_import;
+gpr_cmdline_add_int_type gpr_cmdline_add_int_import;
+gpr_cmdline_add_flag_type gpr_cmdline_add_flag_import;
+gpr_cmdline_add_string_type gpr_cmdline_add_string_import;
+gpr_cmdline_on_extra_arg_type gpr_cmdline_on_extra_arg_import;
+gpr_cmdline_set_survive_failure_type gpr_cmdline_set_survive_failure_import;
+gpr_cmdline_parse_type gpr_cmdline_parse_import;
+gpr_cmdline_destroy_type gpr_cmdline_destroy_import;
+gpr_cmdline_usage_string_type gpr_cmdline_usage_string_import;
+gpr_cpu_num_cores_type gpr_cpu_num_cores_import;
+gpr_cpu_current_cpu_type gpr_cpu_current_cpu_import;
+gpr_histogram_create_type gpr_histogram_create_import;
+gpr_histogram_destroy_type gpr_histogram_destroy_import;
+gpr_histogram_add_type gpr_histogram_add_import;
+gpr_histogram_merge_type gpr_histogram_merge_import;
+gpr_histogram_percentile_type gpr_histogram_percentile_import;
+gpr_histogram_mean_type gpr_histogram_mean_import;
+gpr_histogram_stddev_type gpr_histogram_stddev_import;
+gpr_histogram_variance_type gpr_histogram_variance_import;
+gpr_histogram_maximum_type gpr_histogram_maximum_import;
+gpr_histogram_minimum_type gpr_histogram_minimum_import;
+gpr_histogram_count_type gpr_histogram_count_import;
+gpr_histogram_sum_type gpr_histogram_sum_import;
+gpr_histogram_sum_of_squares_type gpr_histogram_sum_of_squares_import;
+gpr_histogram_get_contents_type gpr_histogram_get_contents_import;
+gpr_histogram_merge_contents_type gpr_histogram_merge_contents_import;
+gpr_join_host_port_type gpr_join_host_port_import;
+gpr_split_host_port_type gpr_split_host_port_import;
 gpr_log_type gpr_log_import;
 gpr_log_message_type gpr_log_message_import;
 gpr_set_log_verbosity_type gpr_set_log_verbosity_import;
 gpr_log_verbosity_init_type gpr_log_verbosity_init_import;
 gpr_set_log_function_type gpr_set_log_function_import;
+gpr_format_message_type gpr_format_message_import;
 gpr_slice_ref_type gpr_slice_ref_import;
 gpr_slice_unref_type gpr_slice_unref_import;
 gpr_slice_new_type gpr_slice_new_import;
@@ -208,6 +245,13 @@ gpr_slice_buffer_move_into_type gpr_slice_buffer_move_into_import;
 gpr_slice_buffer_trim_end_type gpr_slice_buffer_trim_end_import;
 gpr_slice_buffer_move_first_type gpr_slice_buffer_move_first_import;
 gpr_slice_buffer_take_first_type gpr_slice_buffer_take_first_import;
+gpr_strdup_type gpr_strdup_import;
+gpr_asprintf_type gpr_asprintf_import;
+gpr_subprocess_binary_extension_type gpr_subprocess_binary_extension_import;
+gpr_subprocess_create_type gpr_subprocess_create_import;
+gpr_subprocess_destroy_type gpr_subprocess_destroy_import;
+gpr_subprocess_join_type gpr_subprocess_join_import;
+gpr_subprocess_interrupt_type gpr_subprocess_interrupt_import;
 gpr_mu_init_type gpr_mu_init_import;
 gpr_mu_destroy_type gpr_mu_destroy_import;
 gpr_mu_lock_type gpr_mu_lock_import;
@@ -231,6 +275,14 @@ gpr_unref_type gpr_unref_import;
 gpr_stats_init_type gpr_stats_init_import;
 gpr_stats_inc_type gpr_stats_inc_import;
 gpr_stats_read_type gpr_stats_read_import;
+gpr_thd_new_type gpr_thd_new_import;
+gpr_thd_options_default_type gpr_thd_options_default_import;
+gpr_thd_options_set_detached_type gpr_thd_options_set_detached_import;
+gpr_thd_options_set_joinable_type gpr_thd_options_set_joinable_import;
+gpr_thd_options_is_detached_type gpr_thd_options_is_detached_import;
+gpr_thd_options_is_joinable_type gpr_thd_options_is_joinable_import;
+gpr_thd_currentid_type gpr_thd_currentid_import;
+gpr_thd_join_type gpr_thd_join_import;
 gpr_time_0_type gpr_time_0_import;
 gpr_inf_future_type gpr_inf_future_import;
 gpr_inf_past_type gpr_inf_past_import;
@@ -252,60 +304,18 @@ gpr_time_to_millis_type gpr_time_to_millis_import;
 gpr_time_similar_type gpr_time_similar_import;
 gpr_sleep_until_type gpr_sleep_until_import;
 gpr_timespec_to_micros_type gpr_timespec_to_micros_import;
-gpr_avl_create_type gpr_avl_create_import;
-gpr_avl_ref_type gpr_avl_ref_import;
-gpr_avl_unref_type gpr_avl_unref_import;
-gpr_avl_add_type gpr_avl_add_import;
-gpr_avl_remove_type gpr_avl_remove_import;
-gpr_avl_get_type gpr_avl_get_import;
-gpr_avl_maybe_get_type gpr_avl_maybe_get_import;
-gpr_avl_is_empty_type gpr_avl_is_empty_import;
-gpr_cmdline_create_type gpr_cmdline_create_import;
-gpr_cmdline_add_int_type gpr_cmdline_add_int_import;
-gpr_cmdline_add_flag_type gpr_cmdline_add_flag_import;
-gpr_cmdline_add_string_type gpr_cmdline_add_string_import;
-gpr_cmdline_on_extra_arg_type gpr_cmdline_on_extra_arg_import;
-gpr_cmdline_set_survive_failure_type gpr_cmdline_set_survive_failure_import;
-gpr_cmdline_parse_type gpr_cmdline_parse_import;
-gpr_cmdline_destroy_type gpr_cmdline_destroy_import;
-gpr_cmdline_usage_string_type gpr_cmdline_usage_string_import;
-gpr_cpu_num_cores_type gpr_cpu_num_cores_import;
-gpr_cpu_current_cpu_type gpr_cpu_current_cpu_import;
-gpr_histogram_create_type gpr_histogram_create_import;
-gpr_histogram_destroy_type gpr_histogram_destroy_import;
-gpr_histogram_add_type gpr_histogram_add_import;
-gpr_histogram_merge_type gpr_histogram_merge_import;
-gpr_histogram_percentile_type gpr_histogram_percentile_import;
-gpr_histogram_mean_type gpr_histogram_mean_import;
-gpr_histogram_stddev_type gpr_histogram_stddev_import;
-gpr_histogram_variance_type gpr_histogram_variance_import;
-gpr_histogram_maximum_type gpr_histogram_maximum_import;
-gpr_histogram_minimum_type gpr_histogram_minimum_import;
-gpr_histogram_count_type gpr_histogram_count_import;
-gpr_histogram_sum_type gpr_histogram_sum_import;
-gpr_histogram_sum_of_squares_type gpr_histogram_sum_of_squares_import;
-gpr_histogram_get_contents_type gpr_histogram_get_contents_import;
-gpr_histogram_merge_contents_type gpr_histogram_merge_contents_import;
-gpr_join_host_port_type gpr_join_host_port_import;
-gpr_split_host_port_type gpr_split_host_port_import;
-gpr_format_message_type gpr_format_message_import;
-gpr_strdup_type gpr_strdup_import;
-gpr_asprintf_type gpr_asprintf_import;
-gpr_subprocess_binary_extension_type gpr_subprocess_binary_extension_import;
-gpr_subprocess_create_type gpr_subprocess_create_import;
-gpr_subprocess_destroy_type gpr_subprocess_destroy_import;
-gpr_subprocess_join_type gpr_subprocess_join_import;
-gpr_subprocess_interrupt_type gpr_subprocess_interrupt_import;
-gpr_thd_new_type gpr_thd_new_import;
-gpr_thd_options_default_type gpr_thd_options_default_import;
-gpr_thd_options_set_detached_type gpr_thd_options_set_detached_import;
-gpr_thd_options_set_joinable_type gpr_thd_options_set_joinable_import;
-gpr_thd_options_is_detached_type gpr_thd_options_is_detached_import;
-gpr_thd_options_is_joinable_type gpr_thd_options_is_joinable_import;
-gpr_thd_currentid_type gpr_thd_currentid_import;
-gpr_thd_join_type gpr_thd_join_import;
 
 void grpc_rb_load_imports(HMODULE library) {
+  grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create");
+  grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create");
+  grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy");
+  grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length");
+  grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy");
+  grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init");
+  grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy");
+  grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next");
+  grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall");
+  grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader");
   census_initialize_import = (census_initialize_type) GetProcAddress(library, "census_initialize");
   census_shutdown_import = (census_shutdown_type) GetProcAddress(library, "census_shutdown");
   census_supported_import = (census_supported_type) GetProcAddress(library, "census_supported");
@@ -433,21 +443,48 @@ void grpc_rb_load_imports(HMODULE library) {
   gpr_free_aligned_import = (gpr_free_aligned_type) GetProcAddress(library, "gpr_free_aligned");
   gpr_set_allocation_functions_import = (gpr_set_allocation_functions_type) GetProcAddress(library, "gpr_set_allocation_functions");
   gpr_get_allocation_functions_import = (gpr_get_allocation_functions_type) GetProcAddress(library, "gpr_get_allocation_functions");
-  grpc_raw_byte_buffer_create_import = (grpc_raw_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_byte_buffer_create");
-  grpc_raw_compressed_byte_buffer_create_import = (grpc_raw_compressed_byte_buffer_create_type) GetProcAddress(library, "grpc_raw_compressed_byte_buffer_create");
-  grpc_byte_buffer_copy_import = (grpc_byte_buffer_copy_type) GetProcAddress(library, "grpc_byte_buffer_copy");
-  grpc_byte_buffer_length_import = (grpc_byte_buffer_length_type) GetProcAddress(library, "grpc_byte_buffer_length");
-  grpc_byte_buffer_destroy_import = (grpc_byte_buffer_destroy_type) GetProcAddress(library, "grpc_byte_buffer_destroy");
-  grpc_byte_buffer_reader_init_import = (grpc_byte_buffer_reader_init_type) GetProcAddress(library, "grpc_byte_buffer_reader_init");
-  grpc_byte_buffer_reader_destroy_import = (grpc_byte_buffer_reader_destroy_type) GetProcAddress(library, "grpc_byte_buffer_reader_destroy");
-  grpc_byte_buffer_reader_next_import = (grpc_byte_buffer_reader_next_type) GetProcAddress(library, "grpc_byte_buffer_reader_next");
-  grpc_byte_buffer_reader_readall_import = (grpc_byte_buffer_reader_readall_type) GetProcAddress(library, "grpc_byte_buffer_reader_readall");
-  grpc_raw_byte_buffer_from_reader_import = (grpc_raw_byte_buffer_from_reader_type) GetProcAddress(library, "grpc_raw_byte_buffer_from_reader");
+  gpr_avl_create_import = (gpr_avl_create_type) GetProcAddress(library, "gpr_avl_create");
+  gpr_avl_ref_import = (gpr_avl_ref_type) GetProcAddress(library, "gpr_avl_ref");
+  gpr_avl_unref_import = (gpr_avl_unref_type) GetProcAddress(library, "gpr_avl_unref");
+  gpr_avl_add_import = (gpr_avl_add_type) GetProcAddress(library, "gpr_avl_add");
+  gpr_avl_remove_import = (gpr_avl_remove_type) GetProcAddress(library, "gpr_avl_remove");
+  gpr_avl_get_import = (gpr_avl_get_type) GetProcAddress(library, "gpr_avl_get");
+  gpr_avl_maybe_get_import = (gpr_avl_maybe_get_type) GetProcAddress(library, "gpr_avl_maybe_get");
+  gpr_avl_is_empty_import = (gpr_avl_is_empty_type) GetProcAddress(library, "gpr_avl_is_empty");
+  gpr_cmdline_create_import = (gpr_cmdline_create_type) GetProcAddress(library, "gpr_cmdline_create");
+  gpr_cmdline_add_int_import = (gpr_cmdline_add_int_type) GetProcAddress(library, "gpr_cmdline_add_int");
+  gpr_cmdline_add_flag_import = (gpr_cmdline_add_flag_type) GetProcAddress(library, "gpr_cmdline_add_flag");
+  gpr_cmdline_add_string_import = (gpr_cmdline_add_string_type) GetProcAddress(library, "gpr_cmdline_add_string");
+  gpr_cmdline_on_extra_arg_import = (gpr_cmdline_on_extra_arg_type) GetProcAddress(library, "gpr_cmdline_on_extra_arg");
+  gpr_cmdline_set_survive_failure_import = (gpr_cmdline_set_survive_failure_type) GetProcAddress(library, "gpr_cmdline_set_survive_failure");
+  gpr_cmdline_parse_import = (gpr_cmdline_parse_type) GetProcAddress(library, "gpr_cmdline_parse");
+  gpr_cmdline_destroy_import = (gpr_cmdline_destroy_type) GetProcAddress(library, "gpr_cmdline_destroy");
+  gpr_cmdline_usage_string_import = (gpr_cmdline_usage_string_type) GetProcAddress(library, "gpr_cmdline_usage_string");
+  gpr_cpu_num_cores_import = (gpr_cpu_num_cores_type) GetProcAddress(library, "gpr_cpu_num_cores");
+  gpr_cpu_current_cpu_import = (gpr_cpu_current_cpu_type) GetProcAddress(library, "gpr_cpu_current_cpu");
+  gpr_histogram_create_import = (gpr_histogram_create_type) GetProcAddress(library, "gpr_histogram_create");
+  gpr_histogram_destroy_import = (gpr_histogram_destroy_type) GetProcAddress(library, "gpr_histogram_destroy");
+  gpr_histogram_add_import = (gpr_histogram_add_type) GetProcAddress(library, "gpr_histogram_add");
+  gpr_histogram_merge_import = (gpr_histogram_merge_type) GetProcAddress(library, "gpr_histogram_merge");
+  gpr_histogram_percentile_import = (gpr_histogram_percentile_type) GetProcAddress(library, "gpr_histogram_percentile");
+  gpr_histogram_mean_import = (gpr_histogram_mean_type) GetProcAddress(library, "gpr_histogram_mean");
+  gpr_histogram_stddev_import = (gpr_histogram_stddev_type) GetProcAddress(library, "gpr_histogram_stddev");
+  gpr_histogram_variance_import = (gpr_histogram_variance_type) GetProcAddress(library, "gpr_histogram_variance");
+  gpr_histogram_maximum_import = (gpr_histogram_maximum_type) GetProcAddress(library, "gpr_histogram_maximum");
+  gpr_histogram_minimum_import = (gpr_histogram_minimum_type) GetProcAddress(library, "gpr_histogram_minimum");
+  gpr_histogram_count_import = (gpr_histogram_count_type) GetProcAddress(library, "gpr_histogram_count");
+  gpr_histogram_sum_import = (gpr_histogram_sum_type) GetProcAddress(library, "gpr_histogram_sum");
+  gpr_histogram_sum_of_squares_import = (gpr_histogram_sum_of_squares_type) GetProcAddress(library, "gpr_histogram_sum_of_squares");
+  gpr_histogram_get_contents_import = (gpr_histogram_get_contents_type) GetProcAddress(library, "gpr_histogram_get_contents");
+  gpr_histogram_merge_contents_import = (gpr_histogram_merge_contents_type) GetProcAddress(library, "gpr_histogram_merge_contents");
+  gpr_join_host_port_import = (gpr_join_host_port_type) GetProcAddress(library, "gpr_join_host_port");
+  gpr_split_host_port_import = (gpr_split_host_port_type) GetProcAddress(library, "gpr_split_host_port");
   gpr_log_import = (gpr_log_type) GetProcAddress(library, "gpr_log");
   gpr_log_message_import = (gpr_log_message_type) GetProcAddress(library, "gpr_log_message");
   gpr_set_log_verbosity_import = (gpr_set_log_verbosity_type) GetProcAddress(library, "gpr_set_log_verbosity");
   gpr_log_verbosity_init_import = (gpr_log_verbosity_init_type) GetProcAddress(library, "gpr_log_verbosity_init");
   gpr_set_log_function_import = (gpr_set_log_function_type) GetProcAddress(library, "gpr_set_log_function");
+  gpr_format_message_import = (gpr_format_message_type) GetProcAddress(library, "gpr_format_message");
   gpr_slice_ref_import = (gpr_slice_ref_type) GetProcAddress(library, "gpr_slice_ref");
   gpr_slice_unref_import = (gpr_slice_unref_type) GetProcAddress(library, "gpr_slice_unref");
   gpr_slice_new_import = (gpr_slice_new_type) GetProcAddress(library, "gpr_slice_new");
@@ -477,6 +514,13 @@ void grpc_rb_load_imports(HMODULE library) {
   gpr_slice_buffer_trim_end_import = (gpr_slice_buffer_trim_end_type) GetProcAddress(library, "gpr_slice_buffer_trim_end");
   gpr_slice_buffer_move_first_import = (gpr_slice_buffer_move_first_type) GetProcAddress(library, "gpr_slice_buffer_move_first");
   gpr_slice_buffer_take_first_import = (gpr_slice_buffer_take_first_type) GetProcAddress(library, "gpr_slice_buffer_take_first");
+  gpr_strdup_import = (gpr_strdup_type) GetProcAddress(library, "gpr_strdup");
+  gpr_asprintf_import = (gpr_asprintf_type) GetProcAddress(library, "gpr_asprintf");
+  gpr_subprocess_binary_extension_import = (gpr_subprocess_binary_extension_type) GetProcAddress(library, "gpr_subprocess_binary_extension");
+  gpr_subprocess_create_import = (gpr_subprocess_create_type) GetProcAddress(library, "gpr_subprocess_create");
+  gpr_subprocess_destroy_import = (gpr_subprocess_destroy_type) GetProcAddress(library, "gpr_subprocess_destroy");
+  gpr_subprocess_join_import = (gpr_subprocess_join_type) GetProcAddress(library, "gpr_subprocess_join");
+  gpr_subprocess_interrupt_import = (gpr_subprocess_interrupt_type) GetProcAddress(library, "gpr_subprocess_interrupt");
   gpr_mu_init_import = (gpr_mu_init_type) GetProcAddress(library, "gpr_mu_init");
   gpr_mu_destroy_import = (gpr_mu_destroy_type) GetProcAddress(library, "gpr_mu_destroy");
   gpr_mu_lock_import = (gpr_mu_lock_type) GetProcAddress(library, "gpr_mu_lock");
@@ -500,6 +544,14 @@ void grpc_rb_load_imports(HMODULE library) {
   gpr_stats_init_import = (gpr_stats_init_type) GetProcAddress(library, "gpr_stats_init");
   gpr_stats_inc_import = (gpr_stats_inc_type) GetProcAddress(library, "gpr_stats_inc");
   gpr_stats_read_import = (gpr_stats_read_type) GetProcAddress(library, "gpr_stats_read");
+  gpr_thd_new_import = (gpr_thd_new_type) GetProcAddress(library, "gpr_thd_new");
+  gpr_thd_options_default_import = (gpr_thd_options_default_type) GetProcAddress(library, "gpr_thd_options_default");
+  gpr_thd_options_set_detached_import = (gpr_thd_options_set_detached_type) GetProcAddress(library, "gpr_thd_options_set_detached");
+  gpr_thd_options_set_joinable_import = (gpr_thd_options_set_joinable_type) GetProcAddress(library, "gpr_thd_options_set_joinable");
+  gpr_thd_options_is_detached_import = (gpr_thd_options_is_detached_type) GetProcAddress(library, "gpr_thd_options_is_detached");
+  gpr_thd_options_is_joinable_import = (gpr_thd_options_is_joinable_type) GetProcAddress(library, "gpr_thd_options_is_joinable");
+  gpr_thd_currentid_import = (gpr_thd_currentid_type) GetProcAddress(library, "gpr_thd_currentid");
+  gpr_thd_join_import = (gpr_thd_join_type) GetProcAddress(library, "gpr_thd_join");
   gpr_time_0_import = (gpr_time_0_type) GetProcAddress(library, "gpr_time_0");
   gpr_inf_future_import = (gpr_inf_future_type) GetProcAddress(library, "gpr_inf_future");
   gpr_inf_past_import = (gpr_inf_past_type) GetProcAddress(library, "gpr_inf_past");
@@ -521,58 +573,6 @@ void grpc_rb_load_imports(HMODULE library) {
   gpr_time_similar_import = (gpr_time_similar_type) GetProcAddress(library, "gpr_time_similar");
   gpr_sleep_until_import = (gpr_sleep_until_type) GetProcAddress(library, "gpr_sleep_until");
   gpr_timespec_to_micros_import = (gpr_timespec_to_micros_type) GetProcAddress(library, "gpr_timespec_to_micros");
-  gpr_avl_create_import = (gpr_avl_create_type) GetProcAddress(library, "gpr_avl_create");
-  gpr_avl_ref_import = (gpr_avl_ref_type) GetProcAddress(library, "gpr_avl_ref");
-  gpr_avl_unref_import = (gpr_avl_unref_type) GetProcAddress(library, "gpr_avl_unref");
-  gpr_avl_add_import = (gpr_avl_add_type) GetProcAddress(library, "gpr_avl_add");
-  gpr_avl_remove_import = (gpr_avl_remove_type) GetProcAddress(library, "gpr_avl_remove");
-  gpr_avl_get_import = (gpr_avl_get_type) GetProcAddress(library, "gpr_avl_get");
-  gpr_avl_maybe_get_import = (gpr_avl_maybe_get_type) GetProcAddress(library, "gpr_avl_maybe_get");
-  gpr_avl_is_empty_import = (gpr_avl_is_empty_type) GetProcAddress(library, "gpr_avl_is_empty");
-  gpr_cmdline_create_import = (gpr_cmdline_create_type) GetProcAddress(library, "gpr_cmdline_create");
-  gpr_cmdline_add_int_import = (gpr_cmdline_add_int_type) GetProcAddress(library, "gpr_cmdline_add_int");
-  gpr_cmdline_add_flag_import = (gpr_cmdline_add_flag_type) GetProcAddress(library, "gpr_cmdline_add_flag");
-  gpr_cmdline_add_string_import = (gpr_cmdline_add_string_type) GetProcAddress(library, "gpr_cmdline_add_string");
-  gpr_cmdline_on_extra_arg_import = (gpr_cmdline_on_extra_arg_type) GetProcAddress(library, "gpr_cmdline_on_extra_arg");
-  gpr_cmdline_set_survive_failure_import = (gpr_cmdline_set_survive_failure_type) GetProcAddress(library, "gpr_cmdline_set_survive_failure");
-  gpr_cmdline_parse_import = (gpr_cmdline_parse_type) GetProcAddress(library, "gpr_cmdline_parse");
-  gpr_cmdline_destroy_import = (gpr_cmdline_destroy_type) GetProcAddress(library, "gpr_cmdline_destroy");
-  gpr_cmdline_usage_string_import = (gpr_cmdline_usage_string_type) GetProcAddress(library, "gpr_cmdline_usage_string");
-  gpr_cpu_num_cores_import = (gpr_cpu_num_cores_type) GetProcAddress(library, "gpr_cpu_num_cores");
-  gpr_cpu_current_cpu_import = (gpr_cpu_current_cpu_type) GetProcAddress(library, "gpr_cpu_current_cpu");
-  gpr_histogram_create_import = (gpr_histogram_create_type) GetProcAddress(library, "gpr_histogram_create");
-  gpr_histogram_destroy_import = (gpr_histogram_destroy_type) GetProcAddress(library, "gpr_histogram_destroy");
-  gpr_histogram_add_import = (gpr_histogram_add_type) GetProcAddress(library, "gpr_histogram_add");
-  gpr_histogram_merge_import = (gpr_histogram_merge_type) GetProcAddress(library, "gpr_histogram_merge");
-  gpr_histogram_percentile_import = (gpr_histogram_percentile_type) GetProcAddress(library, "gpr_histogram_percentile");
-  gpr_histogram_mean_import = (gpr_histogram_mean_type) GetProcAddress(library, "gpr_histogram_mean");
-  gpr_histogram_stddev_import = (gpr_histogram_stddev_type) GetProcAddress(library, "gpr_histogram_stddev");
-  gpr_histogram_variance_import = (gpr_histogram_variance_type) GetProcAddress(library, "gpr_histogram_variance");
-  gpr_histogram_maximum_import = (gpr_histogram_maximum_type) GetProcAddress(library, "gpr_histogram_maximum");
-  gpr_histogram_minimum_import = (gpr_histogram_minimum_type) GetProcAddress(library, "gpr_histogram_minimum");
-  gpr_histogram_count_import = (gpr_histogram_count_type) GetProcAddress(library, "gpr_histogram_count");
-  gpr_histogram_sum_import = (gpr_histogram_sum_type) GetProcAddress(library, "gpr_histogram_sum");
-  gpr_histogram_sum_of_squares_import = (gpr_histogram_sum_of_squares_type) GetProcAddress(library, "gpr_histogram_sum_of_squares");
-  gpr_histogram_get_contents_import = (gpr_histogram_get_contents_type) GetProcAddress(library, "gpr_histogram_get_contents");
-  gpr_histogram_merge_contents_import = (gpr_histogram_merge_contents_type) GetProcAddress(library, "gpr_histogram_merge_contents");
-  gpr_join_host_port_import = (gpr_join_host_port_type) GetProcAddress(library, "gpr_join_host_port");
-  gpr_split_host_port_import = (gpr_split_host_port_type) GetProcAddress(library, "gpr_split_host_port");
-  gpr_format_message_import = (gpr_format_message_type) GetProcAddress(library, "gpr_format_message");
-  gpr_strdup_import = (gpr_strdup_type) GetProcAddress(library, "gpr_strdup");
-  gpr_asprintf_import = (gpr_asprintf_type) GetProcAddress(library, "gpr_asprintf");
-  gpr_subprocess_binary_extension_import = (gpr_subprocess_binary_extension_type) GetProcAddress(library, "gpr_subprocess_binary_extension");
-  gpr_subprocess_create_import = (gpr_subprocess_create_type) GetProcAddress(library, "gpr_subprocess_create");
-  gpr_subprocess_destroy_import = (gpr_subprocess_destroy_type) GetProcAddress(library, "gpr_subprocess_destroy");
-  gpr_subprocess_join_import = (gpr_subprocess_join_type) GetProcAddress(library, "gpr_subprocess_join");
-  gpr_subprocess_interrupt_import = (gpr_subprocess_interrupt_type) GetProcAddress(library, "gpr_subprocess_interrupt");
-  gpr_thd_new_import = (gpr_thd_new_type) GetProcAddress(library, "gpr_thd_new");
-  gpr_thd_options_default_import = (gpr_thd_options_default_type) GetProcAddress(library, "gpr_thd_options_default");
-  gpr_thd_options_set_detached_import = (gpr_thd_options_set_detached_type) GetProcAddress(library, "gpr_thd_options_set_detached");
-  gpr_thd_options_set_joinable_import = (gpr_thd_options_set_joinable_type) GetProcAddress(library, "gpr_thd_options_set_joinable");
-  gpr_thd_options_is_detached_import = (gpr_thd_options_is_detached_type) GetProcAddress(library, "gpr_thd_options_is_detached");
-  gpr_thd_options_is_joinable_import = (gpr_thd_options_is_joinable_type) GetProcAddress(library, "gpr_thd_options_is_joinable");
-  gpr_thd_currentid_import = (gpr_thd_currentid_type) GetProcAddress(library, "gpr_thd_currentid");
-  gpr_thd_join_import = (gpr_thd_join_type) GetProcAddress(library, "gpr_thd_join");
 }
 
 #endif /* GPR_WINDOWS */
diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h
index a2f5b86497..b87abfd8ef 100644
--- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h
+++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h
@@ -40,28 +40,58 @@
 
 #include <windows.h>
 
+#include <grpc/byte_buffer.h>
 #include <grpc/census.h>
 #include <grpc/compression.h>
 #include <grpc/grpc.h>
 #include <grpc/grpc_posix.h>
 #include <grpc/grpc_security.h>
-#include <grpc/impl/codegen/alloc.h>
-#include <grpc/impl/codegen/byte_buffer.h>
-#include <grpc/impl/codegen/log.h>
-#include <grpc/impl/codegen/slice.h>
-#include <grpc/impl/codegen/slice_buffer.h>
-#include <grpc/impl/codegen/sync.h>
-#include <grpc/impl/codegen/time.h>
+#include <grpc/support/alloc.h>
 #include <grpc/support/avl.h>
 #include <grpc/support/cmdline.h>
 #include <grpc/support/cpu.h>
 #include <grpc/support/histogram.h>
 #include <grpc/support/host_port.h>
+#include <grpc/support/log.h>
 #include <grpc/support/log_windows.h>
+#include <grpc/support/slice.h>
+#include <grpc/support/slice_buffer.h>
 #include <grpc/support/string_util.h>
 #include <grpc/support/subprocess.h>
+#include <grpc/support/sync.h>
 #include <grpc/support/thd.h>
+#include <grpc/support/time.h>
 
+typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(gpr_slice *slices, size_t nslices);
+extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import;
+#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import
+typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(gpr_slice *slices, size_t nslices, grpc_compression_algorithm compression);
+extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import;
+#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import
+typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb);
+extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import;
+#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import
+typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb);
+extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import;
+#define grpc_byte_buffer_length grpc_byte_buffer_length_import
+typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer);
+extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import;
+#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import
+typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer);
+extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import;
+#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import
+typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader);
+extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import;
+#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import
+typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, gpr_slice *slice);
+extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import;
+#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import
+typedef gpr_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader);
+extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import;
+#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import
+typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader);
+extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import;
+#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import
 typedef int(*census_initialize_type)(int features);
 extern census_initialize_type census_initialize_import;
 #define census_initialize census_initialize_import
@@ -443,36 +473,114 @@ extern gpr_set_allocation_functions_type gpr_set_allocation_functions_import;
 typedef gpr_allocation_functions(*gpr_get_allocation_functions_type)();
 extern gpr_get_allocation_functions_type gpr_get_allocation_functions_import;
 #define gpr_get_allocation_functions gpr_get_allocation_functions_import
-typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_create_type)(gpr_slice *slices, size_t nslices);
-extern grpc_raw_byte_buffer_create_type grpc_raw_byte_buffer_create_import;
-#define grpc_raw_byte_buffer_create grpc_raw_byte_buffer_create_import
-typedef grpc_byte_buffer *(*grpc_raw_compressed_byte_buffer_create_type)(gpr_slice *slices, size_t nslices, grpc_compression_algorithm compression);
-extern grpc_raw_compressed_byte_buffer_create_type grpc_raw_compressed_byte_buffer_create_import;
-#define grpc_raw_compressed_byte_buffer_create grpc_raw_compressed_byte_buffer_create_import
-typedef grpc_byte_buffer *(*grpc_byte_buffer_copy_type)(grpc_byte_buffer *bb);
-extern grpc_byte_buffer_copy_type grpc_byte_buffer_copy_import;
-#define grpc_byte_buffer_copy grpc_byte_buffer_copy_import
-typedef size_t(*grpc_byte_buffer_length_type)(grpc_byte_buffer *bb);
-extern grpc_byte_buffer_length_type grpc_byte_buffer_length_import;
-#define grpc_byte_buffer_length grpc_byte_buffer_length_import
-typedef void(*grpc_byte_buffer_destroy_type)(grpc_byte_buffer *byte_buffer);
-extern grpc_byte_buffer_destroy_type grpc_byte_buffer_destroy_import;
-#define grpc_byte_buffer_destroy grpc_byte_buffer_destroy_import
-typedef int(*grpc_byte_buffer_reader_init_type)(grpc_byte_buffer_reader *reader, grpc_byte_buffer *buffer);
-extern grpc_byte_buffer_reader_init_type grpc_byte_buffer_reader_init_import;
-#define grpc_byte_buffer_reader_init grpc_byte_buffer_reader_init_import
-typedef void(*grpc_byte_buffer_reader_destroy_type)(grpc_byte_buffer_reader *reader);
-extern grpc_byte_buffer_reader_destroy_type grpc_byte_buffer_reader_destroy_import;
-#define grpc_byte_buffer_reader_destroy grpc_byte_buffer_reader_destroy_import
-typedef int(*grpc_byte_buffer_reader_next_type)(grpc_byte_buffer_reader *reader, gpr_slice *slice);
-extern grpc_byte_buffer_reader_next_type grpc_byte_buffer_reader_next_import;
-#define grpc_byte_buffer_reader_next grpc_byte_buffer_reader_next_import
-typedef gpr_slice(*grpc_byte_buffer_reader_readall_type)(grpc_byte_buffer_reader *reader);
-extern grpc_byte_buffer_reader_readall_type grpc_byte_buffer_reader_readall_import;
-#define grpc_byte_buffer_reader_readall grpc_byte_buffer_reader_readall_import
-typedef grpc_byte_buffer *(*grpc_raw_byte_buffer_from_reader_type)(grpc_byte_buffer_reader *reader);
-extern grpc_raw_byte_buffer_from_reader_type grpc_raw_byte_buffer_from_reader_import;
-#define grpc_raw_byte_buffer_from_reader grpc_raw_byte_buffer_from_reader_import
+typedef gpr_avl(*gpr_avl_create_type)(const gpr_avl_vtable *vtable);
+extern gpr_avl_create_type gpr_avl_create_import;
+#define gpr_avl_create gpr_avl_create_import
+typedef gpr_avl(*gpr_avl_ref_type)(gpr_avl avl);
+extern gpr_avl_ref_type gpr_avl_ref_import;
+#define gpr_avl_ref gpr_avl_ref_import
+typedef void(*gpr_avl_unref_type)(gpr_avl avl);
+extern gpr_avl_unref_type gpr_avl_unref_import;
+#define gpr_avl_unref gpr_avl_unref_import
+typedef gpr_avl(*gpr_avl_add_type)(gpr_avl avl, void *key, void *value);
+extern gpr_avl_add_type gpr_avl_add_import;
+#define gpr_avl_add gpr_avl_add_import
+typedef gpr_avl(*gpr_avl_remove_type)(gpr_avl avl, void *key);
+extern gpr_avl_remove_type gpr_avl_remove_import;
+#define gpr_avl_remove gpr_avl_remove_import
+typedef void *(*gpr_avl_get_type)(gpr_avl avl, void *key);
+extern gpr_avl_get_type gpr_avl_get_import;
+#define gpr_avl_get gpr_avl_get_import
+typedef int(*gpr_avl_maybe_get_type)(gpr_avl avl, void *key, void **value);
+extern gpr_avl_maybe_get_type gpr_avl_maybe_get_import;
+#define gpr_avl_maybe_get gpr_avl_maybe_get_import
+typedef int(*gpr_avl_is_empty_type)(gpr_avl avl);
+extern gpr_avl_is_empty_type gpr_avl_is_empty_import;
+#define gpr_avl_is_empty gpr_avl_is_empty_import
+typedef gpr_cmdline *(*gpr_cmdline_create_type)(const char *description);
+extern gpr_cmdline_create_type gpr_cmdline_create_import;
+#define gpr_cmdline_create gpr_cmdline_create_import
+typedef void(*gpr_cmdline_add_int_type)(gpr_cmdline *cl, const char *name, const char *help, int *value);
+extern gpr_cmdline_add_int_type gpr_cmdline_add_int_import;
+#define gpr_cmdline_add_int gpr_cmdline_add_int_import
+typedef void(*gpr_cmdline_add_flag_type)(gpr_cmdline *cl, const char *name, const char *help, int *value);
+extern gpr_cmdline_add_flag_type gpr_cmdline_add_flag_import;
+#define gpr_cmdline_add_flag gpr_cmdline_add_flag_import
+typedef void(*gpr_cmdline_add_string_type)(gpr_cmdline *cl, const char *name, const char *help, char **value);
+extern gpr_cmdline_add_string_type gpr_cmdline_add_string_import;
+#define gpr_cmdline_add_string gpr_cmdline_add_string_import
+typedef void(*gpr_cmdline_on_extra_arg_type)(gpr_cmdline *cl, const char *name, const char *help, void (*on_extra_arg)(void *user_data, const char *arg), void *user_data);
+extern gpr_cmdline_on_extra_arg_type gpr_cmdline_on_extra_arg_import;
+#define gpr_cmdline_on_extra_arg gpr_cmdline_on_extra_arg_import
+typedef void(*gpr_cmdline_set_survive_failure_type)(gpr_cmdline *cl);
+extern gpr_cmdline_set_survive_failure_type gpr_cmdline_set_survive_failure_import;
+#define gpr_cmdline_set_survive_failure gpr_cmdline_set_survive_failure_import
+typedef int(*gpr_cmdline_parse_type)(gpr_cmdline *cl, int argc, char **argv);
+extern gpr_cmdline_parse_type gpr_cmdline_parse_import;
+#define gpr_cmdline_parse gpr_cmdline_parse_import
+typedef void(*gpr_cmdline_destroy_type)(gpr_cmdline *cl);
+extern gpr_cmdline_destroy_type gpr_cmdline_destroy_import;
+#define gpr_cmdline_destroy gpr_cmdline_destroy_import
+typedef char *(*gpr_cmdline_usage_string_type)(gpr_cmdline *cl, const char *argv0);
+extern gpr_cmdline_usage_string_type gpr_cmdline_usage_string_import;
+#define gpr_cmdline_usage_string gpr_cmdline_usage_string_import
+typedef unsigned(*gpr_cpu_num_cores_type)(void);
+extern gpr_cpu_num_cores_type gpr_cpu_num_cores_import;
+#define gpr_cpu_num_cores gpr_cpu_num_cores_import
+typedef unsigned(*gpr_cpu_current_cpu_type)(void);
+extern gpr_cpu_current_cpu_type gpr_cpu_current_cpu_import;
+#define gpr_cpu_current_cpu gpr_cpu_current_cpu_import
+typedef gpr_histogram *(*gpr_histogram_create_type)(double resolution, double max_bucket_start);
+extern gpr_histogram_create_type gpr_histogram_create_import;
+#define gpr_histogram_create gpr_histogram_create_import
+typedef void(*gpr_histogram_destroy_type)(gpr_histogram *h);
+extern gpr_histogram_destroy_type gpr_histogram_destroy_import;
+#define gpr_histogram_destroy gpr_histogram_destroy_import
+typedef void(*gpr_histogram_add_type)(gpr_histogram *h, double x);
+extern gpr_histogram_add_type gpr_histogram_add_import;
+#define gpr_histogram_add gpr_histogram_add_import
+typedef int(*gpr_histogram_merge_type)(gpr_histogram *dst, const gpr_histogram *src);
+extern gpr_histogram_merge_type gpr_histogram_merge_import;
+#define gpr_histogram_merge gpr_histogram_merge_import
+typedef double(*gpr_histogram_percentile_type)(gpr_histogram *histogram, double percentile);
+extern gpr_histogram_percentile_type gpr_histogram_percentile_import;
+#define gpr_histogram_percentile gpr_histogram_percentile_import
+typedef double(*gpr_histogram_mean_type)(gpr_histogram *histogram);
+extern gpr_histogram_mean_type gpr_histogram_mean_import;
+#define gpr_histogram_mean gpr_histogram_mean_import
+typedef double(*gpr_histogram_stddev_type)(gpr_histogram *histogram);
+extern gpr_histogram_stddev_type gpr_histogram_stddev_import;
+#define gpr_histogram_stddev gpr_histogram_stddev_import
+typedef double(*gpr_histogram_variance_type)(gpr_histogram *histogram);
+extern gpr_histogram_variance_type gpr_histogram_variance_import;
+#define gpr_histogram_variance gpr_histogram_variance_import
+typedef double(*gpr_histogram_maximum_type)(gpr_histogram *histogram);
+extern gpr_histogram_maximum_type gpr_histogram_maximum_import;
+#define gpr_histogram_maximum gpr_histogram_maximum_import
+typedef double(*gpr_histogram_minimum_type)(gpr_histogram *histogram);
+extern gpr_histogram_minimum_type gpr_histogram_minimum_import;
+#define gpr_histogram_minimum gpr_histogram_minimum_import
+typedef double(*gpr_histogram_count_type)(gpr_histogram *histogram);
+extern gpr_histogram_count_type gpr_histogram_count_import;
+#define gpr_histogram_count gpr_histogram_count_import
+typedef double(*gpr_histogram_sum_type)(gpr_histogram *histogram);
+extern gpr_histogram_sum_type gpr_histogram_sum_import;
+#define gpr_histogram_sum gpr_histogram_sum_import
+typedef double(*gpr_histogram_sum_of_squares_type)(gpr_histogram *histogram);
+extern gpr_histogram_sum_of_squares_type gpr_histogram_sum_of_squares_import;
+#define gpr_histogram_sum_of_squares gpr_histogram_sum_of_squares_import
+typedef const uint32_t *(*gpr_histogram_get_contents_type)(gpr_histogram *histogram, size_t *count);
+extern gpr_histogram_get_contents_type gpr_histogram_get_contents_import;
+#define gpr_histogram_get_contents gpr_histogram_get_contents_import
+typedef void(*gpr_histogram_merge_contents_type)(gpr_histogram *histogram, const uint32_t *data, size_t data_count, double min_seen, double max_seen, double sum, double sum_of_squares, double count);
+extern gpr_histogram_merge_contents_type gpr_histogram_merge_contents_import;
+#define gpr_histogram_merge_contents gpr_histogram_merge_contents_import
+typedef int(*gpr_join_host_port_type)(char **out, const char *host, int port);
+extern gpr_join_host_port_type gpr_join_host_port_import;
+#define gpr_join_host_port gpr_join_host_port_import
+typedef int(*gpr_split_host_port_type)(const char *name, char **host, char **port);
+extern gpr_split_host_port_type gpr_split_host_port_import;
+#define gpr_split_host_port gpr_split_host_port_import
 typedef void(*gpr_log_type)(const char *file, int line, gpr_log_severity severity, const char *format, ...) GPRC_PRINT_FORMAT_CHECK(4, 5);
 extern gpr_log_type gpr_log_import;
 #define gpr_log gpr_log_import
@@ -488,6 +596,9 @@ extern gpr_log_verbosity_init_type gpr_log_verbosity_init_import;
 typedef void(*gpr_set_log_function_type)(gpr_log_func func);
 extern gpr_set_log_function_type gpr_set_log_function_import;
 #define gpr_set_log_function gpr_set_log_function_import
+typedef char *(*gpr_format_message_type)(int messageid);
+extern gpr_format_message_type gpr_format_message_import;
+#define gpr_format_message gpr_format_message_import
 typedef gpr_slice(*gpr_slice_ref_type)(gpr_slice s);
 extern gpr_slice_ref_type gpr_slice_ref_import;
 #define gpr_slice_ref gpr_slice_ref_import
@@ -575,6 +686,27 @@ extern gpr_slice_buffer_move_first_type gpr_slice_buffer_move_first_import;
 typedef gpr_slice(*gpr_slice_buffer_take_first_type)(gpr_slice_buffer *src);
 extern gpr_slice_buffer_take_first_type gpr_slice_buffer_take_first_import;
 #define gpr_slice_buffer_take_first gpr_slice_buffer_take_first_import
+typedef char *(*gpr_strdup_type)(const char *src);
+extern gpr_strdup_type gpr_strdup_import;
+#define gpr_strdup gpr_strdup_import
+typedef int(*gpr_asprintf_type)(char **strp, const char *format, ...) GPRC_PRINT_FORMAT_CHECK(2, 3);
+extern gpr_asprintf_type gpr_asprintf_import;
+#define gpr_asprintf gpr_asprintf_import
+typedef const char *(*gpr_subprocess_binary_extension_type)();
+extern gpr_subprocess_binary_extension_type gpr_subprocess_binary_extension_import;
+#define gpr_subprocess_binary_extension gpr_subprocess_binary_extension_import
+typedef gpr_subprocess *(*gpr_subprocess_create_type)(int argc, const char **argv);
+extern gpr_subprocess_create_type gpr_subprocess_create_import;
+#define gpr_subprocess_create gpr_subprocess_create_import
+typedef void(*gpr_subprocess_destroy_type)(gpr_subprocess *p);
+extern gpr_subprocess_destroy_type gpr_subprocess_destroy_import;
+#define gpr_subprocess_destroy gpr_subprocess_destroy_import
+typedef int(*gpr_subprocess_join_type)(gpr_subprocess *p);
+extern gpr_subprocess_join_type gpr_subprocess_join_import;
+#define gpr_subprocess_join gpr_subprocess_join_import
+typedef void(*gpr_subprocess_interrupt_type)(gpr_subprocess *p);
+extern gpr_subprocess_interrupt_type gpr_subprocess_interrupt_import;
+#define gpr_subprocess_interrupt gpr_subprocess_interrupt_import
 typedef void(*gpr_mu_init_type)(gpr_mu *mu);
 extern gpr_mu_init_type gpr_mu_init_import;
 #define gpr_mu_init gpr_mu_init_import
@@ -644,6 +776,30 @@ extern gpr_stats_inc_type gpr_stats_inc_import;
 typedef intptr_t(*gpr_stats_read_type)(const gpr_stats_counter *c);
 extern gpr_stats_read_type gpr_stats_read_import;
 #define gpr_stats_read gpr_stats_read_import
+typedef int(*gpr_thd_new_type)(gpr_thd_id *t, void (*thd_body)(void *arg), void *arg, const gpr_thd_options *options);
+extern gpr_thd_new_type gpr_thd_new_import;
+#define gpr_thd_new gpr_thd_new_import
+typedef gpr_thd_options(*gpr_thd_options_default_type)(void);
+extern gpr_thd_options_default_type gpr_thd_options_default_import;
+#define gpr_thd_options_default gpr_thd_options_default_import
+typedef void(*gpr_thd_options_set_detached_type)(gpr_thd_options *options);
+extern gpr_thd_options_set_detached_type gpr_thd_options_set_detached_import;
+#define gpr_thd_options_set_detached gpr_thd_options_set_detached_import
+typedef void(*gpr_thd_options_set_joinable_type)(gpr_thd_options *options);
+extern gpr_thd_options_set_joinable_type gpr_thd_options_set_joinable_import;
+#define gpr_thd_options_set_joinable gpr_thd_options_set_joinable_import
+typedef int(*gpr_thd_options_is_detached_type)(const gpr_thd_options *options);
+extern gpr_thd_options_is_detached_type gpr_thd_options_is_detached_import;
+#define gpr_thd_options_is_detached gpr_thd_options_is_detached_import
+typedef int(*gpr_thd_options_is_joinable_type)(const gpr_thd_options *options);
+extern gpr_thd_options_is_joinable_type gpr_thd_options_is_joinable_import;
+#define gpr_thd_options_is_joinable gpr_thd_options_is_joinable_import
+typedef gpr_thd_id(*gpr_thd_currentid_type)(void);
+extern gpr_thd_currentid_type gpr_thd_currentid_import;
+#define gpr_thd_currentid gpr_thd_currentid_import
+typedef void(*gpr_thd_join_type)(gpr_thd_id t);
+extern gpr_thd_join_type gpr_thd_join_import;
+#define gpr_thd_join gpr_thd_join_import
 typedef gpr_timespec(*gpr_time_0_type)(gpr_clock_type type);
 extern gpr_time_0_type gpr_time_0_import;
 #define gpr_time_0 gpr_time_0_import
@@ -707,162 +863,6 @@ extern gpr_sleep_until_type gpr_sleep_until_import;
 typedef double(*gpr_timespec_to_micros_type)(gpr_timespec t);
 extern gpr_timespec_to_micros_type gpr_timespec_to_micros_import;
 #define gpr_timespec_to_micros gpr_timespec_to_micros_import
-typedef gpr_avl(*gpr_avl_create_type)(const gpr_avl_vtable *vtable);
-extern gpr_avl_create_type gpr_avl_create_import;
-#define gpr_avl_create gpr_avl_create_import
-typedef gpr_avl(*gpr_avl_ref_type)(gpr_avl avl);
-extern gpr_avl_ref_type gpr_avl_ref_import;
-#define gpr_avl_ref gpr_avl_ref_import
-typedef void(*gpr_avl_unref_type)(gpr_avl avl);
-extern gpr_avl_unref_type gpr_avl_unref_import;
-#define gpr_avl_unref gpr_avl_unref_import
-typedef gpr_avl(*gpr_avl_add_type)(gpr_avl avl, void *key, void *value);
-extern gpr_avl_add_type gpr_avl_add_import;
-#define gpr_avl_add gpr_avl_add_import
-typedef gpr_avl(*gpr_avl_remove_type)(gpr_avl avl, void *key);
-extern gpr_avl_remove_type gpr_avl_remove_import;
-#define gpr_avl_remove gpr_avl_remove_import
-typedef void *(*gpr_avl_get_type)(gpr_avl avl, void *key);
-extern gpr_avl_get_type gpr_avl_get_import;
-#define gpr_avl_get gpr_avl_get_import
-typedef int(*gpr_avl_maybe_get_type)(gpr_avl avl, void *key, void **value);
-extern gpr_avl_maybe_get_type gpr_avl_maybe_get_import;
-#define gpr_avl_maybe_get gpr_avl_maybe_get_import
-typedef int(*gpr_avl_is_empty_type)(gpr_avl avl);
-extern gpr_avl_is_empty_type gpr_avl_is_empty_import;
-#define gpr_avl_is_empty gpr_avl_is_empty_import
-typedef gpr_cmdline *(*gpr_cmdline_create_type)(const char *description);
-extern gpr_cmdline_create_type gpr_cmdline_create_import;
-#define gpr_cmdline_create gpr_cmdline_create_import
-typedef void(*gpr_cmdline_add_int_type)(gpr_cmdline *cl, const char *name, const char *help, int *value);
-extern gpr_cmdline_add_int_type gpr_cmdline_add_int_import;
-#define gpr_cmdline_add_int gpr_cmdline_add_int_import
-typedef void(*gpr_cmdline_add_flag_type)(gpr_cmdline *cl, const char *name, const char *help, int *value);
-extern gpr_cmdline_add_flag_type gpr_cmdline_add_flag_import;
-#define gpr_cmdline_add_flag gpr_cmdline_add_flag_import
-typedef void(*gpr_cmdline_add_string_type)(gpr_cmdline *cl, const char *name, const char *help, char **value);
-extern gpr_cmdline_add_string_type gpr_cmdline_add_string_import;
-#define gpr_cmdline_add_string gpr_cmdline_add_string_import
-typedef void(*gpr_cmdline_on_extra_arg_type)(gpr_cmdline *cl, const char *name, const char *help, void (*on_extra_arg)(void *user_data, const char *arg), void *user_data);
-extern gpr_cmdline_on_extra_arg_type gpr_cmdline_on_extra_arg_import;
-#define gpr_cmdline_on_extra_arg gpr_cmdline_on_extra_arg_import
-typedef void(*gpr_cmdline_set_survive_failure_type)(gpr_cmdline *cl);
-extern gpr_cmdline_set_survive_failure_type gpr_cmdline_set_survive_failure_import;
-#define gpr_cmdline_set_survive_failure gpr_cmdline_set_survive_failure_import
-typedef int(*gpr_cmdline_parse_type)(gpr_cmdline *cl, int argc, char **argv);
-extern gpr_cmdline_parse_type gpr_cmdline_parse_import;
-#define gpr_cmdline_parse gpr_cmdline_parse_import
-typedef void(*gpr_cmdline_destroy_type)(gpr_cmdline *cl);
-extern gpr_cmdline_destroy_type gpr_cmdline_destroy_import;
-#define gpr_cmdline_destroy gpr_cmdline_destroy_import
-typedef char *(*gpr_cmdline_usage_string_type)(gpr_cmdline *cl, const char *argv0);
-extern gpr_cmdline_usage_string_type gpr_cmdline_usage_string_import;
-#define gpr_cmdline_usage_string gpr_cmdline_usage_string_import
-typedef unsigned(*gpr_cpu_num_cores_type)(void);
-extern gpr_cpu_num_cores_type gpr_cpu_num_cores_import;
-#define gpr_cpu_num_cores gpr_cpu_num_cores_import
-typedef unsigned(*gpr_cpu_current_cpu_type)(void);
-extern gpr_cpu_current_cpu_type gpr_cpu_current_cpu_import;
-#define gpr_cpu_current_cpu gpr_cpu_current_cpu_import
-typedef gpr_histogram *(*gpr_histogram_create_type)(double resolution, double max_bucket_start);
-extern gpr_histogram_create_type gpr_histogram_create_import;
-#define gpr_histogram_create gpr_histogram_create_import
-typedef void(*gpr_histogram_destroy_type)(gpr_histogram *h);
-extern gpr_histogram_destroy_type gpr_histogram_destroy_import;
-#define gpr_histogram_destroy gpr_histogram_destroy_import
-typedef void(*gpr_histogram_add_type)(gpr_histogram *h, double x);
-extern gpr_histogram_add_type gpr_histogram_add_import;
-#define gpr_histogram_add gpr_histogram_add_import
-typedef int(*gpr_histogram_merge_type)(gpr_histogram *dst, const gpr_histogram *src);
-extern gpr_histogram_merge_type gpr_histogram_merge_import;
-#define gpr_histogram_merge gpr_histogram_merge_import
-typedef double(*gpr_histogram_percentile_type)(gpr_histogram *histogram, double percentile);
-extern gpr_histogram_percentile_type gpr_histogram_percentile_import;
-#define gpr_histogram_percentile gpr_histogram_percentile_import
-typedef double(*gpr_histogram_mean_type)(gpr_histogram *histogram);
-extern gpr_histogram_mean_type gpr_histogram_mean_import;
-#define gpr_histogram_mean gpr_histogram_mean_import
-typedef double(*gpr_histogram_stddev_type)(gpr_histogram *histogram);
-extern gpr_histogram_stddev_type gpr_histogram_stddev_import;
-#define gpr_histogram_stddev gpr_histogram_stddev_import
-typedef double(*gpr_histogram_variance_type)(gpr_histogram *histogram);
-extern gpr_histogram_variance_type gpr_histogram_variance_import;
-#define gpr_histogram_variance gpr_histogram_variance_import
-typedef double(*gpr_histogram_maximum_type)(gpr_histogram *histogram);
-extern gpr_histogram_maximum_type gpr_histogram_maximum_import;
-#define gpr_histogram_maximum gpr_histogram_maximum_import
-typedef double(*gpr_histogram_minimum_type)(gpr_histogram *histogram);
-extern gpr_histogram_minimum_type gpr_histogram_minimum_import;
-#define gpr_histogram_minimum gpr_histogram_minimum_import
-typedef double(*gpr_histogram_count_type)(gpr_histogram *histogram);
-extern gpr_histogram_count_type gpr_histogram_count_import;
-#define gpr_histogram_count gpr_histogram_count_import
-typedef double(*gpr_histogram_sum_type)(gpr_histogram *histogram);
-extern gpr_histogram_sum_type gpr_histogram_sum_import;
-#define gpr_histogram_sum gpr_histogram_sum_import
-typedef double(*gpr_histogram_sum_of_squares_type)(gpr_histogram *histogram);
-extern gpr_histogram_sum_of_squares_type gpr_histogram_sum_of_squares_import;
-#define gpr_histogram_sum_of_squares gpr_histogram_sum_of_squares_import
-typedef const uint32_t *(*gpr_histogram_get_contents_type)(gpr_histogram *histogram, size_t *count);
-extern gpr_histogram_get_contents_type gpr_histogram_get_contents_import;
-#define gpr_histogram_get_contents gpr_histogram_get_contents_import
-typedef void(*gpr_histogram_merge_contents_type)(gpr_histogram *histogram, const uint32_t *data, size_t data_count, double min_seen, double max_seen, double sum, double sum_of_squares, double count);
-extern gpr_histogram_merge_contents_type gpr_histogram_merge_contents_import;
-#define gpr_histogram_merge_contents gpr_histogram_merge_contents_import
-typedef int(*gpr_join_host_port_type)(char **out, const char *host, int port);
-extern gpr_join_host_port_type gpr_join_host_port_import;
-#define gpr_join_host_port gpr_join_host_port_import
-typedef int(*gpr_split_host_port_type)(const char *name, char **host, char **port);
-extern gpr_split_host_port_type gpr_split_host_port_import;
-#define gpr_split_host_port gpr_split_host_port_import
-typedef char *(*gpr_format_message_type)(int messageid);
-extern gpr_format_message_type gpr_format_message_import;
-#define gpr_format_message gpr_format_message_import
-typedef char *(*gpr_strdup_type)(const char *src);
-extern gpr_strdup_type gpr_strdup_import;
-#define gpr_strdup gpr_strdup_import
-typedef int(*gpr_asprintf_type)(char **strp, const char *format, ...) GPRC_PRINT_FORMAT_CHECK(2, 3);
-extern gpr_asprintf_type gpr_asprintf_import;
-#define gpr_asprintf gpr_asprintf_import
-typedef const char *(*gpr_subprocess_binary_extension_type)();
-extern gpr_subprocess_binary_extension_type gpr_subprocess_binary_extension_import;
-#define gpr_subprocess_binary_extension gpr_subprocess_binary_extension_import
-typedef gpr_subprocess *(*gpr_subprocess_create_type)(int argc, const char **argv);
-extern gpr_subprocess_create_type gpr_subprocess_create_import;
-#define gpr_subprocess_create gpr_subprocess_create_import
-typedef void(*gpr_subprocess_destroy_type)(gpr_subprocess *p);
-extern gpr_subprocess_destroy_type gpr_subprocess_destroy_import;
-#define gpr_subprocess_destroy gpr_subprocess_destroy_import
-typedef int(*gpr_subprocess_join_type)(gpr_subprocess *p);
-extern gpr_subprocess_join_type gpr_subprocess_join_import;
-#define gpr_subprocess_join gpr_subprocess_join_import
-typedef void(*gpr_subprocess_interrupt_type)(gpr_subprocess *p);
-extern gpr_subprocess_interrupt_type gpr_subprocess_interrupt_import;
-#define gpr_subprocess_interrupt gpr_subprocess_interrupt_import
-typedef int(*gpr_thd_new_type)(gpr_thd_id *t, void (*thd_body)(void *arg), void *arg, const gpr_thd_options *options);
-extern gpr_thd_new_type gpr_thd_new_import;
-#define gpr_thd_new gpr_thd_new_import
-typedef gpr_thd_options(*gpr_thd_options_default_type)(void);
-extern gpr_thd_options_default_type gpr_thd_options_default_import;
-#define gpr_thd_options_default gpr_thd_options_default_import
-typedef void(*gpr_thd_options_set_detached_type)(gpr_thd_options *options);
-extern gpr_thd_options_set_detached_type gpr_thd_options_set_detached_import;
-#define gpr_thd_options_set_detached gpr_thd_options_set_detached_import
-typedef void(*gpr_thd_options_set_joinable_type)(gpr_thd_options *options);
-extern gpr_thd_options_set_joinable_type gpr_thd_options_set_joinable_import;
-#define gpr_thd_options_set_joinable gpr_thd_options_set_joinable_import
-typedef int(*gpr_thd_options_is_detached_type)(const gpr_thd_options *options);
-extern gpr_thd_options_is_detached_type gpr_thd_options_is_detached_import;
-#define gpr_thd_options_is_detached gpr_thd_options_is_detached_import
-typedef int(*gpr_thd_options_is_joinable_type)(const gpr_thd_options *options);
-extern gpr_thd_options_is_joinable_type gpr_thd_options_is_joinable_import;
-#define gpr_thd_options_is_joinable gpr_thd_options_is_joinable_import
-typedef gpr_thd_id(*gpr_thd_currentid_type)(void);
-extern gpr_thd_currentid_type gpr_thd_currentid_import;
-#define gpr_thd_currentid gpr_thd_currentid_import
-typedef void(*gpr_thd_join_type)(gpr_thd_id t);
-extern gpr_thd_join_type gpr_thd_join_import;
-#define gpr_thd_join gpr_thd_join_import
 
 void grpc_rb_load_imports(HMODULE library);