Merge branch 'master' of github.com:grpc/grpc into authority_header

15 files changed, 355 insertions, 107 deletions

diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc
index c63de3c509..aca7307a2f 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc
@@ -28,6 +28,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/string_util.h>
 
+#include <address_sorting/address_sorting.h>
+
 #include "src/core/ext/filters/client_channel/http_connect_handshaker.h"
 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
 #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h"
@@ -458,6 +460,7 @@ void grpc_resolver_dns_ares_init() {
   /* TODO(zyc): Turn on c-ares based resolver by default after the address
      sorter and the CNAME support are added. */
   if (resolver_env != nullptr && gpr_stricmp(resolver_env, "ares") == 0) {
+    address_sorting_init();
     grpc_error* error = grpc_ares_init();
     if (error != GRPC_ERROR_NONE) {
       GRPC_LOG_IF_ERROR("ares_library_init() failed", error);
@@ -475,6 +478,7 @@ void grpc_resolver_dns_ares_init() {
 void grpc_resolver_dns_ares_shutdown() {
   char* resolver_env = gpr_getenv("GRPC_DNS_RESOLVER");
   if (resolver_env != nullptr && gpr_stricmp(resolver_env, "ares") == 0) {
+    address_sorting_shutdown();
     grpc_ares_cleanup();
   }
   gpr_free(resolver_env);
diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc
index 71b06eb87e..fb2435749d 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc
@@ -33,6 +33,7 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 
+#include <address_sorting/address_sorting.h>
 #include "src/core/ext/filters/client_channel/parse_address.h"
 #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h"
 #include "src/core/lib/gpr/host_port.h"
@@ -46,6 +47,9 @@
 static gpr_once g_basic_init = GPR_ONCE_INIT;
 static gpr_mu g_init_mu;
 
+grpc_core::TraceFlag grpc_trace_cares_address_sorting(false,
+                                                      "cares_address_sorting");
+
 struct grpc_ares_request {
   /** indicates the DNS server to use, if specified */
   struct ares_addr_port_node dns_server_addr;
@@ -96,11 +100,60 @@ static void grpc_ares_request_ref(grpc_ares_request* r) {
   gpr_ref(&r->pending_queries);
 }
 
+static void log_address_sorting_list(grpc_lb_addresses* lb_addrs,
+                                     const char* input_output_str) {
+  for (size_t i = 0; i < lb_addrs->num_addresses; i++) {
+    char* addr_str;
+    if (grpc_sockaddr_to_string(&addr_str, &lb_addrs->addresses[i].address,
+                                true)) {
+      gpr_log(GPR_DEBUG, "c-ares address sorting: %s[%" PRIuPTR "]=%s",
+              input_output_str, i, addr_str);
+      gpr_free(addr_str);
+    } else {
+      gpr_log(GPR_DEBUG,
+              "c-ares address sorting: %s[%" PRIuPTR "]=<unprintable>",
+              input_output_str, i);
+    }
+  }
+}
+
+void grpc_cares_wrapper_address_sorting_sort(grpc_lb_addresses* lb_addrs) {
+  if (grpc_trace_cares_address_sorting.enabled()) {
+    log_address_sorting_list(lb_addrs, "input");
+  }
+  address_sorting_sortable* sortables = (address_sorting_sortable*)gpr_zalloc(
+      sizeof(address_sorting_sortable) * lb_addrs->num_addresses);
+  for (size_t i = 0; i < lb_addrs->num_addresses; i++) {
+    sortables[i].user_data = &lb_addrs->addresses[i];
+    memcpy(&sortables[i].dest_addr.addr, &lb_addrs->addresses[i].address.addr,
+           lb_addrs->addresses[i].address.len);
+    sortables[i].dest_addr.len = lb_addrs->addresses[i].address.len;
+  }
+  address_sorting_rfc_6724_sort(sortables, lb_addrs->num_addresses);
+  grpc_lb_address* sorted_lb_addrs = (grpc_lb_address*)gpr_zalloc(
+      sizeof(grpc_lb_address) * lb_addrs->num_addresses);
+  for (size_t i = 0; i < lb_addrs->num_addresses; i++) {
+    sorted_lb_addrs[i] = *(grpc_lb_address*)sortables[i].user_data;
+  }
+  gpr_free(sortables);
+  gpr_free(lb_addrs->addresses);
+  lb_addrs->addresses = sorted_lb_addrs;
+  if (grpc_trace_cares_address_sorting.enabled()) {
+    log_address_sorting_list(lb_addrs, "output");
+  }
+}
+
+/* Allow tests to access grpc_ares_wrapper_address_sorting_sort */
+void grpc_cares_wrapper_test_only_address_sorting_sort(
+    grpc_lb_addresses* lb_addrs) {
+  grpc_cares_wrapper_address_sorting_sort(lb_addrs);
+}
+
 static void grpc_ares_request_unref(grpc_ares_request* r) {
   /* If there are no pending queries, invoke on_done callback and destroy the
      request */
   if (gpr_unref(&r->pending_queries)) {
-    /* TODO(zyc): Sort results with RFC6724 before invoking on_done. */
+    grpc_cares_wrapper_address_sorting_sort(*(r->lb_addrs_out));
     GRPC_CLOSURE_SCHED(r->on_done, r->error);
     gpr_mu_destroy(&r->mu);
     grpc_ares_ev_driver_destroy(r->ev_driver);
diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h
index 3e8ea01e12..2d84a038d6 100644
--- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h
+++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h
@@ -26,6 +26,8 @@
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 
+extern grpc_core::TraceFlag grpc_trace_cares_address_sorting;
+
 typedef struct grpc_ares_request grpc_ares_request;
 
 /* Asynchronously resolve \a name. Use \a default_port if a port isn't
@@ -64,5 +66,9 @@ grpc_error* grpc_ares_init(void);
    it has been called the same number of times as grpc_ares_init(). */
 void grpc_ares_cleanup(void);
 
+/* Exposed only for testing */
+void grpc_cares_wrapper_test_only_address_sorting_sort(
+    grpc_lb_addresses* lb_addrs);
+
 #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_DNS_C_ARES_GRPC_ARES_WRAPPER_H \
         */
diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index ab69cecf3a..dc4e002405 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -669,6 +669,7 @@ static int init_stream(grpc_transport* gt, grpc_stream* gs,
   GRPC_CLOSURE_INIT(&s->complete_fetch_locked, complete_fetch_locked, s,
                     grpc_schedule_on_exec_ctx);
   grpc_slice_buffer_init(&s->unprocessed_incoming_frames_buffer);
+  s->unprocessed_incoming_frames_buffer_cached_length = 0;
   grpc_slice_buffer_init(&s->frame_storage);
   grpc_slice_buffer_init(&s->compressed_data_buffer);
   grpc_slice_buffer_init(&s->decompressed_data_buffer);
@@ -1577,20 +1578,27 @@ static void perform_stream_op_locked(void* stream_op,
 
   if (op->recv_message) {
     GRPC_STATS_INC_HTTP2_OP_RECV_MESSAGE();
-    size_t already_received;
+    size_t before = 0;
     GPR_ASSERT(s->recv_message_ready == nullptr);
     GPR_ASSERT(!s->pending_byte_stream);
     s->recv_message_ready = op_payload->recv_message.recv_message_ready;
     s->recv_message = op_payload->recv_message.recv_message;
     if (s->id != 0) {
       if (!s->read_closed) {
-        already_received = s->frame_storage.length;
+        before = s->frame_storage.length +
+                 s->unprocessed_incoming_frames_buffer.length;
+      }
+    }
+    grpc_chttp2_maybe_complete_recv_message(t, s);
+    if (s->id != 0) {
+      if (!s->read_closed && s->frame_storage.length == 0) {
+        size_t after = s->frame_storage.length +
+                       s->unprocessed_incoming_frames_buffer_cached_length;
         s->flow_control->IncomingByteStreamUpdate(GRPC_HEADER_SIZE_IN_BYTES,
-                                                  already_received);
+                                                  before - after);
         grpc_chttp2_act_on_flowctl_action(s->flow_control->MakeAction(), t, s);
       }
     }
-    grpc_chttp2_maybe_complete_recv_message(t, s);
   }
 
   if (op->recv_trailing_metadata) {
@@ -1867,6 +1875,10 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* t,
         }
       }
     }
+    // save the length of the buffer before handing control back to application
+    // threads. Needed to support correct flow control bookkeeping
+    s->unprocessed_incoming_frames_buffer_cached_length =
+        s->unprocessed_incoming_frames_buffer.length;
     if (error == GRPC_ERROR_NONE && *s->recv_message != nullptr) {
       null_then_run_closure(&s->recv_message_ready, GRPC_ERROR_NONE);
     } else if (s->published_metadata[1] != GRPC_METADATA_NOT_PUBLISHED) {
diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h
index 6d11e5aa31..ca6e715978 100644
--- a/src/core/ext/transport/chttp2/transport/internal.h
+++ b/src/core/ext/transport/chttp2/transport/internal.h
@@ -550,6 +550,11 @@ struct grpc_chttp2_stream {
   grpc_slice_buffer unprocessed_incoming_frames_buffer;
   grpc_closure* on_next;    /* protected by t combiner */
   bool pending_byte_stream; /* protected by t combiner */
+  // cached length of buffer to be used by the transport thread in cases where
+  // stream->pending_byte_stream == true. The value is saved before
+  // application threads are allowed to modify
+  // unprocessed_incoming_frames_buffer
+  size_t unprocessed_incoming_frames_buffer_cached_length;
   grpc_closure reset_byte_stream;
   grpc_error* byte_stream_error; /* protected by t combiner */
   bool received_last_frame;      /* protected by t combiner */
diff --git a/src/core/lib/http/httpcli_security_connector.cc b/src/core/lib/http/httpcli_security_connector.cc
index 886357fc00..0b53d63e77 100644
--- a/src/core/lib/http/httpcli_security_connector.cc
+++ b/src/core/lib/http/httpcli_security_connector.cc
@@ -102,8 +102,8 @@ static grpc_security_connector_vtable httpcli_ssl_vtable = {
     httpcli_ssl_destroy, httpcli_ssl_check_peer, httpcli_ssl_cmp};
 
 static grpc_security_status httpcli_ssl_channel_security_connector_create(
-    const char* pem_root_certs, const char* secure_peer_name,
-    grpc_channel_security_connector** sc) {
+    const char* pem_root_certs, const tsi_ssl_root_certs_store* root_store,
+    const char* secure_peer_name, grpc_channel_security_connector** sc) {
   tsi_result result = TSI_OK;
   grpc_httpcli_ssl_channel_security_connector* c;
 
@@ -124,6 +124,7 @@ static grpc_security_status httpcli_ssl_channel_security_connector_create(
   tsi_ssl_client_handshaker_options options;
   memset(&options, 0, sizeof(options));
   options.pem_root_certs = pem_root_certs;
+  options.root_store = root_store;
   result = tsi_create_ssl_client_handshaker_factory_with_options(
       &options, &c->handshaker_factory);
   if (result != TSI_OK) {
@@ -172,8 +173,11 @@ static void ssl_handshake(void* arg, grpc_endpoint* tcp, const char* host,
                           grpc_millis deadline,
                           void (*on_done)(void* arg, grpc_endpoint* endpoint)) {
   on_done_closure* c = static_cast<on_done_closure*>(gpr_malloc(sizeof(*c)));
-  const char* pem_root_certs = grpc_get_default_ssl_roots();
-  if (pem_root_certs == nullptr) {
+  const char* pem_root_certs =
+      grpc_core::DefaultSslRootStore::GetPemRootCerts();
+  const tsi_ssl_root_certs_store* root_store =
+      grpc_core::DefaultSslRootStore::GetRootStore();
+  if (root_store == nullptr) {
     gpr_log(GPR_ERROR, "Could not get default pem root certs.");
     on_done(arg, nullptr);
     gpr_free(c);
@@ -183,7 +187,7 @@ static void ssl_handshake(void* arg, grpc_endpoint* tcp, const char* host,
   c->arg = arg;
   grpc_channel_security_connector* sc = nullptr;
   GPR_ASSERT(httpcli_ssl_channel_security_connector_create(
-                 pem_root_certs, host, &sc) == GRPC_SECURITY_OK);
+                 pem_root_certs, root_store, host, &sc) == GRPC_SECURITY_OK);
   grpc_arg channel_arg = grpc_security_connector_to_arg(&sc->base);
   grpc_channel_args args = {1, &channel_arg};
   c->handshake_mgr = grpc_handshake_manager_create();
diff --git a/src/core/lib/security/security_connector/security_connector.cc b/src/core/lib/security/security_connector/security_connector.cc
index 980c9e9248..d7d5a8c4de 100644
--- a/src/core/lib/security/security_connector/security_connector.cc
+++ b/src/core/lib/security/security_connector/security_connector.cc
@@ -996,63 +996,6 @@ static grpc_security_connector_vtable ssl_channel_vtable = {
 static grpc_security_connector_vtable ssl_server_vtable = {
     ssl_server_destroy, ssl_server_check_peer, ssl_server_cmp};
 
-/* returns a NULL terminated slice. */
-static grpc_slice compute_default_pem_root_certs_once(void) {
-  grpc_slice result = grpc_empty_slice();
-
-  /* First try to load the roots from the environment. */
-  char* default_root_certs_path =
-      gpr_getenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR);
-  if (default_root_certs_path != nullptr) {
-    GRPC_LOG_IF_ERROR("load_file",
-                      grpc_load_file(default_root_certs_path, 1, &result));
-    gpr_free(default_root_certs_path);
-  }
-
-  /* Try overridden roots if needed. */
-  grpc_ssl_roots_override_result ovrd_res = GRPC_SSL_ROOTS_OVERRIDE_FAIL;
-  if (GRPC_SLICE_IS_EMPTY(result) && ssl_roots_override_cb != nullptr) {
-    char* pem_root_certs = nullptr;
-    ovrd_res = ssl_roots_override_cb(&pem_root_certs);
-    if (ovrd_res == GRPC_SSL_ROOTS_OVERRIDE_OK) {
-      GPR_ASSERT(pem_root_certs != nullptr);
-      result = grpc_slice_from_copied_buffer(
-          pem_root_certs,
-          strlen(pem_root_certs) + 1);  // NULL terminator.
-    }
-    gpr_free(pem_root_certs);
-  }
-
-  /* Fall back to installed certs if needed. */
-  if (GRPC_SLICE_IS_EMPTY(result) &&
-      ovrd_res != GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY) {
-    GRPC_LOG_IF_ERROR("load_file",
-                      grpc_load_file(installed_roots_path, 1, &result));
-  }
-  return result;
-}
-
-static grpc_slice default_pem_root_certs;
-
-static void init_default_pem_root_certs(void) {
-  default_pem_root_certs = compute_default_pem_root_certs_once();
-}
-
-grpc_slice grpc_get_default_ssl_roots_for_testing(void) {
-  return compute_default_pem_root_certs_once();
-}
-
-const char* grpc_get_default_ssl_roots(void) {
-  /* TODO(jboeuf@google.com): Maybe revisit the approach which consists in
-     loading all the roots once for the lifetime of the process. */
-  static gpr_once once = GPR_ONCE_INIT;
-  gpr_once_init(&once, init_default_pem_root_certs);
-  return GRPC_SLICE_IS_EMPTY(default_pem_root_certs)
-             ? nullptr
-             : reinterpret_cast<const char*>
-                   GRPC_SLICE_START_PTR(default_pem_root_certs);
-}
-
 grpc_security_status grpc_ssl_channel_security_connector_create(
     grpc_channel_credentials* channel_creds,
     grpc_call_credentials* request_metadata_creds,
@@ -1074,7 +1017,9 @@ grpc_security_status grpc_ssl_channel_security_connector_create(
     goto error;
   }
   if (config->pem_root_certs == nullptr) {
-    options.pem_root_certs = grpc_get_default_ssl_roots();
+    // Use default root certificates.
+    options.pem_root_certs = grpc_core::DefaultSslRootStore::GetPemRootCerts();
+    options.root_store = grpc_core::DefaultSslRootStore::GetRootStore();
     if (options.pem_root_certs == nullptr) {
       gpr_log(GPR_ERROR, "Could not get default pem root certs.");
       goto error;
@@ -1082,7 +1027,6 @@ grpc_security_status grpc_ssl_channel_security_connector_create(
   } else {
     options.pem_root_certs = config->pem_root_certs;
   }
-
   c = static_cast<grpc_ssl_channel_security_connector*>(
       gpr_zalloc(sizeof(grpc_ssl_channel_security_connector)));
 
@@ -1188,3 +1132,79 @@ grpc_security_status grpc_ssl_server_security_connector_create(
   }
   return retval;
 }
+
+namespace grpc_core {
+
+tsi_ssl_root_certs_store* DefaultSslRootStore::default_root_store_;
+grpc_slice DefaultSslRootStore::default_pem_root_certs_;
+
+const tsi_ssl_root_certs_store* DefaultSslRootStore::GetRootStore() {
+  InitRootStore();
+  return default_root_store_;
+}
+
+const char* DefaultSslRootStore::GetPemRootCerts() {
+  InitRootStore();
+  return GRPC_SLICE_IS_EMPTY(default_pem_root_certs_)
+             ? nullptr
+             : reinterpret_cast<const char*>
+                   GRPC_SLICE_START_PTR(default_pem_root_certs_);
+}
+
+void DefaultSslRootStore::Initialize() {
+  default_root_store_ = nullptr;
+  default_pem_root_certs_ = grpc_empty_slice();
+}
+
+void DefaultSslRootStore::Destroy() {
+  tsi_ssl_root_certs_store_destroy(default_root_store_);
+  grpc_slice_unref_internal(default_pem_root_certs_);
+}
+
+grpc_slice DefaultSslRootStore::ComputePemRootCerts() {
+  grpc_slice result = grpc_empty_slice();
+  // First try to load the roots from the environment.
+  char* default_root_certs_path =
+      gpr_getenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR);
+  if (default_root_certs_path != nullptr) {
+    GRPC_LOG_IF_ERROR("load_file",
+                      grpc_load_file(default_root_certs_path, 1, &result));
+    gpr_free(default_root_certs_path);
+  }
+  // Try overridden roots if needed.
+  grpc_ssl_roots_override_result ovrd_res = GRPC_SSL_ROOTS_OVERRIDE_FAIL;
+  if (GRPC_SLICE_IS_EMPTY(result) && ssl_roots_override_cb != nullptr) {
+    char* pem_root_certs = nullptr;
+    ovrd_res = ssl_roots_override_cb(&pem_root_certs);
+    if (ovrd_res == GRPC_SSL_ROOTS_OVERRIDE_OK) {
+      GPR_ASSERT(pem_root_certs != nullptr);
+      result = grpc_slice_from_copied_buffer(
+          pem_root_certs,
+          strlen(pem_root_certs) + 1);  // nullptr terminator.
+    }
+    gpr_free(pem_root_certs);
+  }
+  // Fall back to installed certs if needed.
+  if (GRPC_SLICE_IS_EMPTY(result) &&
+      ovrd_res != GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY) {
+    GRPC_LOG_IF_ERROR("load_file",
+                      grpc_load_file(installed_roots_path, 1, &result));
+  }
+  return result;
+}
+
+void DefaultSslRootStore::InitRootStore() {
+  static gpr_once once = GPR_ONCE_INIT;
+  gpr_once_init(&once, DefaultSslRootStore::InitRootStoreOnce);
+}
+
+void DefaultSslRootStore::InitRootStoreOnce() {
+  default_pem_root_certs_ = ComputePemRootCerts();
+  if (!GRPC_SLICE_IS_EMPTY(default_pem_root_certs_)) {
+    default_root_store_ =
+        tsi_ssl_root_certs_store_create(reinterpret_cast<const char*>(
+            GRPC_SLICE_START_PTR(default_pem_root_certs_)));
+  }
+}
+
+}  // namespace grpc_core
diff --git a/src/core/lib/security/security_connector/security_connector.h b/src/core/lib/security/security_connector/security_connector.h
index dc847d94f9..5d3d1e0f44 100644
--- a/src/core/lib/security/security_connector/security_connector.h
+++ b/src/core/lib/security/security_connector/security_connector.h
@@ -216,12 +216,6 @@ grpc_security_status grpc_ssl_channel_security_connector_create(
     tsi_ssl_session_cache* ssl_session_cache,
     grpc_channel_security_connector** sc);
 
-/* Gets the default ssl roots. Returns NULL if not found. */
-const char* grpc_get_default_ssl_roots(void);
-
-/* Exposed for TESTING ONLY!. */
-grpc_slice grpc_get_default_ssl_roots_for_testing(void);
-
 /* Config for ssl servers. */
 typedef struct {
   tsi_ssl_pem_key_cert_pair* pem_key_cert_pairs;
@@ -250,4 +244,49 @@ tsi_peer tsi_shallow_peer_from_ssl_auth_context(
     const grpc_auth_context* auth_context);
 void tsi_shallow_peer_destruct(tsi_peer* peer);
 
+/* --- Default SSL Root Store. --- */
+namespace grpc_core {
+
+// The class implements default SSL root store.
+class DefaultSslRootStore {
+ public:
+  // Gets the default SSL root store. Returns nullptr if not found.
+  static const tsi_ssl_root_certs_store* GetRootStore();
+
+  // Gets the default PEM root certificate.
+  static const char* GetPemRootCerts();
+
+  // Initializes the SSL root store's underlying data structure. It does not
+  // load default SSL root certificates. Should only be called by
+  // grpc_security_init().
+  static void Initialize();
+
+  // Destroys the default SSL root store. Should only be called by
+  // grpc_security_shutdown().
+  static void Destroy();
+
+ protected:
+  // Returns default PEM root certificates in nullptr terminated grpc_slice.
+  // This function is protected instead of private, so that it can be tested.
+  static grpc_slice ComputePemRootCerts();
+
+ private:
+  // Construct me not!
+  DefaultSslRootStore();
+
+  // Initialization of default SSL root store.
+  static void InitRootStore();
+
+  // One-time initialization of default SSL root store.
+  static void InitRootStoreOnce();
+
+  // SSL root store in tsi_ssl_root_certs_store object.
+  static tsi_ssl_root_certs_store* default_root_store_;
+
+  // Default PEM root certificates.
+  static grpc_slice default_pem_root_certs_;
+};
+
+}  // namespace grpc_core
+
 #endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_SECURITY_CONNECTOR_H */
diff --git a/src/core/lib/surface/init.cc b/src/core/lib/surface/init.cc
index bd436d6857..52e0ee1c44 100644
--- a/src/core/lib/surface/init.cc
+++ b/src/core/lib/surface/init.cc
@@ -172,6 +172,7 @@ void grpc_shutdown(void) {
           }
         }
       }
+      grpc_security_shutdown();
       grpc_iomgr_shutdown();
       gpr_timers_global_destroy();
       grpc_tracer_shutdown();
diff --git a/src/core/lib/surface/init.h b/src/core/lib/surface/init.h
index 9353208332..d8282b475b 100644
--- a/src/core/lib/surface/init.h
+++ b/src/core/lib/surface/init.h
@@ -22,6 +22,7 @@
 void grpc_register_security_filters(void);
 void grpc_security_pre_init(void);
 void grpc_security_init(void);
+void grpc_security_shutdown(void);
 int grpc_is_initialized(void);
 
 #endif /* GRPC_CORE_LIB_SURFACE_INIT_H */
diff --git a/src/core/lib/surface/init_secure.cc b/src/core/lib/surface/init_secure.cc
index 28c6f7b121..caa67c2512 100644
--- a/src/core/lib/surface/init_secure.cc
+++ b/src/core/lib/surface/init_secure.cc
@@ -78,4 +78,9 @@ void grpc_register_security_filters(void) {
                                    maybe_prepend_server_auth_filter, nullptr);
 }
 
-void grpc_security_init() { grpc_security_register_handshaker_factories(); }
+void grpc_security_init() {
+  grpc_security_register_handshaker_factories();
+  grpc_core::DefaultSslRootStore::Initialize();
+}
+
+void grpc_security_shutdown() { grpc_core::DefaultSslRootStore::Destroy(); }
diff --git a/src/core/lib/surface/init_unsecure.cc b/src/core/lib/surface/init_unsecure.cc
index 2b3bc64382..1c8d07b38b 100644
--- a/src/core/lib/surface/init_unsecure.cc
+++ b/src/core/lib/surface/init_unsecure.cc
@@ -25,3 +25,5 @@ void grpc_security_pre_init(void) {}
 void grpc_register_security_filters(void) {}
 
 void grpc_security_init(void) {}
+
+void grpc_security_shutdown(void) {}
diff --git a/src/core/plugin_registry/grpc_cronet_plugin_registry.cc b/src/core/plugin_registry/grpc_cronet_plugin_registry.cc
index ab0f4994ea..84228cb38a 100644
--- a/src/core/plugin_registry/grpc_cronet_plugin_registry.cc
+++ b/src/core/plugin_registry/grpc_cronet_plugin_registry.cc
@@ -22,32 +22,48 @@
 
 void grpc_deadline_filter_init(void);
 void grpc_deadline_filter_shutdown(void);
+void grpc_client_channel_init(void);
+void grpc_client_channel_shutdown(void);
+void grpc_lb_policy_pick_first_init(void);
+void grpc_lb_policy_pick_first_shutdown(void);
+void grpc_max_age_filter_init(void);
+void grpc_max_age_filter_shutdown(void);
 void grpc_message_size_filter_init(void);
 void grpc_message_size_filter_shutdown(void);
+void grpc_resolver_dns_native_init(void);
+void grpc_resolver_dns_native_shutdown(void);
+void grpc_resolver_sockaddr_init(void);
+void grpc_resolver_sockaddr_shutdown(void);
 void grpc_server_load_reporting_plugin_init(void);
 void grpc_server_load_reporting_plugin_shutdown(void);
 void grpc_http_filters_init(void);
 void grpc_http_filters_shutdown(void);
 void grpc_chttp2_plugin_init(void);
 void grpc_chttp2_plugin_shutdown(void);
-void grpc_client_channel_init(void);
-void grpc_client_channel_shutdown(void);
 void grpc_tsi_alts_init(void);
 void grpc_tsi_alts_shutdown(void);
 
 void grpc_register_built_in_plugins(void) {
   grpc_register_plugin(grpc_deadline_filter_init,
                        grpc_deadline_filter_shutdown);
+  grpc_register_plugin(grpc_client_channel_init,
+                       grpc_client_channel_shutdown);
+  grpc_register_plugin(grpc_lb_policy_pick_first_init,
+                       grpc_lb_policy_pick_first_shutdown);
+  grpc_register_plugin(grpc_max_age_filter_init,
+                       grpc_max_age_filter_shutdown);
   grpc_register_plugin(grpc_message_size_filter_init,
                        grpc_message_size_filter_shutdown);
+  grpc_register_plugin(grpc_resolver_dns_native_init,
+                       grpc_resolver_dns_native_shutdown);
+  grpc_register_plugin(grpc_resolver_sockaddr_init,
+                       grpc_resolver_sockaddr_shutdown);
   grpc_register_plugin(grpc_server_load_reporting_plugin_init,
                        grpc_server_load_reporting_plugin_shutdown);
   grpc_register_plugin(grpc_http_filters_init,
                        grpc_http_filters_shutdown);
   grpc_register_plugin(grpc_chttp2_plugin_init,
                        grpc_chttp2_plugin_shutdown);
-  grpc_register_plugin(grpc_client_channel_init,
-                       grpc_client_channel_shutdown);
   grpc_register_plugin(grpc_tsi_alts_init,
                        grpc_tsi_alts_shutdown);
 }
diff --git a/src/core/tsi/ssl_transport_security.cc b/src/core/tsi/ssl_transport_security.cc
index 0fc2926cf6..0ba6587678 100644
--- a/src/core/tsi/ssl_transport_security.cc
+++ b/src/core/tsi/ssl_transport_security.cc
@@ -71,6 +71,10 @@ extern "C" {
 
 /* --- Structure definitions. ---*/
 
+struct tsi_ssl_root_certs_store {
+  X509_STORE* store;
+};
+
 struct tsi_ssl_handshaker_factory {
   const tsi_ssl_handshaker_factory_vtable* vtable;
   gpr_refcount refcount;
@@ -553,21 +557,18 @@ static tsi_result ssl_ctx_use_private_key(SSL_CTX* context, const char* pem_key,
 
 /* Loads in-memory PEM verification certs into the SSL context and optionally
    returns the verification cert names (root_names can be NULL). */
-static tsi_result ssl_ctx_load_verification_certs(SSL_CTX* context,
-                                                  const char* pem_roots,
-                                                  size_t pem_roots_size,
-                                                  STACK_OF(X509_NAME) *
-                                                      *root_names) {
+static tsi_result x509_store_load_certs(X509_STORE* cert_store,
+                                        const char* pem_roots,
+                                        size_t pem_roots_size,
+                                        STACK_OF(X509_NAME) * *root_names) {
   tsi_result result = TSI_OK;
   size_t num_roots = 0;
   X509* root = nullptr;
   X509_NAME* root_name = nullptr;
   BIO* pem;
-  X509_STORE* root_store;
   GPR_ASSERT(pem_roots_size <= INT_MAX);
   pem = BIO_new_mem_buf((void*)pem_roots, static_cast<int>(pem_roots_size));
-  root_store = SSL_CTX_get_cert_store(context);
-  if (root_store == nullptr) return TSI_INVALID_ARGUMENT;
+  if (cert_store == nullptr) return TSI_INVALID_ARGUMENT;
   if (pem == nullptr) return TSI_OUT_OF_RESOURCES;
   if (root_names != nullptr) {
     *root_names = sk_X509_NAME_new_null();
@@ -595,7 +596,7 @@ static tsi_result ssl_ctx_load_verification_certs(SSL_CTX* context,
       sk_X509_NAME_push(*root_names, root_name);
       root_name = nullptr;
     }
-    if (!X509_STORE_add_cert(root_store, root)) {
+    if (!X509_STORE_add_cert(cert_store, root)) {
       gpr_log(GPR_ERROR, "Could not add root certificate to ssl context.");
       result = TSI_INTERNAL_ERROR;
       break;
@@ -621,6 +622,16 @@ static tsi_result ssl_ctx_load_verification_certs(SSL_CTX* context,
   return result;
 }
 
+static tsi_result ssl_ctx_load_verification_certs(SSL_CTX* context,
+                                                  const char* pem_roots,
+                                                  size_t pem_roots_size,
+                                                  STACK_OF(X509_NAME) *
+                                                      *root_name) {
+  X509_STORE* cert_store = SSL_CTX_get_cert_store(context);
+  return x509_store_load_certs(cert_store, pem_roots, pem_roots_size,
+                               root_name);
+}
+
 /* Populates the SSL context with a private key and a cert chain, and sets the
    cipher list and the ephemeral ECDH key. */
 static tsi_result populate_ssl_context(
@@ -730,6 +741,43 @@ static int NullVerifyCallback(int preverify_ok, X509_STORE_CTX* ctx) {
   return 1;
 }
 
+/* --- tsi_ssl_root_certs_store methods implementation. ---*/
+
+tsi_ssl_root_certs_store* tsi_ssl_root_certs_store_create(
+    const char* pem_roots) {
+  if (pem_roots == nullptr) {
+    gpr_log(GPR_ERROR, "The root certificates are empty.");
+    return nullptr;
+  }
+  tsi_ssl_root_certs_store* root_store = static_cast<tsi_ssl_root_certs_store*>(
+      gpr_zalloc(sizeof(tsi_ssl_root_certs_store)));
+  if (root_store == nullptr) {
+    gpr_log(GPR_ERROR, "Could not allocate buffer for ssl_root_certs_store.");
+    return nullptr;
+  }
+  root_store->store = X509_STORE_new();
+  if (root_store->store == nullptr) {
+    gpr_log(GPR_ERROR, "Could not allocate buffer for X509_STORE.");
+    gpr_free(root_store);
+    return nullptr;
+  }
+  tsi_result result = x509_store_load_certs(root_store->store, pem_roots,
+                                            strlen(pem_roots), nullptr);
+  if (result != TSI_OK) {
+    gpr_log(GPR_ERROR, "Could not load root certificates.");
+    X509_STORE_free(root_store->store);
+    gpr_free(root_store);
+    return nullptr;
+  }
+  return root_store;
+}
+
+void tsi_ssl_root_certs_store_destroy(tsi_ssl_root_certs_store* self) {
+  if (self == nullptr) return;
+  X509_STORE_free(self->store);
+  gpr_free(self);
+}
+
 /* --- tsi_ssl_session_cache methods implementation. ---*/
 
 tsi_ssl_session_cache* tsi_ssl_session_cache_create_lru(size_t capacity) {
@@ -1468,7 +1516,9 @@ tsi_result tsi_create_ssl_client_handshaker_factory_with_options(
 
   if (factory == nullptr) return TSI_INVALID_ARGUMENT;
   *factory = nullptr;
-  if (options->pem_root_certs == nullptr) return TSI_INVALID_ARGUMENT;
+  if (options->pem_root_certs == nullptr && options->root_store == nullptr) {
+    return TSI_INVALID_ARGUMENT;
+  }
 
   ssl_context = SSL_CTX_new(TLSv1_2_method());
   if (ssl_context == nullptr) {
@@ -1480,9 +1530,7 @@ tsi_result tsi_create_ssl_client_handshaker_factory_with_options(
       gpr_zalloc(sizeof(*impl)));
   tsi_ssl_handshaker_factory_init(&impl->base);
   impl->base.vtable = &client_handshaker_factory_vtable;
-
   impl->ssl_context = ssl_context;
-
   if (options->session_cache != nullptr) {
     // Unref is called manually on factory destruction.
     impl->session_cache =
@@ -1498,12 +1546,22 @@ tsi_result tsi_create_ssl_client_handshaker_factory_with_options(
     result = populate_ssl_context(ssl_context, options->pem_key_cert_pair,
                                   options->cipher_suites);
     if (result != TSI_OK) break;
-    result = ssl_ctx_load_verification_certs(
-        ssl_context, options->pem_root_certs, strlen(options->pem_root_certs),
-        nullptr);
-    if (result != TSI_OK) {
-      gpr_log(GPR_ERROR, "Cannot load server root certificates.");
-      break;
+
+#if OPENSSL_VERSION_NUMBER >= 0x10100000
+    // X509_STORE_up_ref is only available since OpenSSL 1.1.
+    if (options->root_store != nullptr) {
+      X509_STORE_up_ref(options->root_store->store);
+      SSL_CTX_set_cert_store(ssl_context, options->root_store->store);
+    }
+#endif
+    if (OPENSSL_VERSION_NUMBER < 0x10100000 || options->root_store == nullptr) {
+      result = ssl_ctx_load_verification_certs(
+          ssl_context, options->pem_root_certs, strlen(options->pem_root_certs),
+          nullptr);
+      if (result != TSI_OK) {
+        gpr_log(GPR_ERROR, "Cannot load server root certificates.");
+        break;
+      }
     }
 
     if (options->num_alpn_protocols != 0) {
diff --git a/src/core/tsi/ssl_transport_security.h b/src/core/tsi/ssl_transport_security.h
index 29d209b8f5..cabf583098 100644
--- a/src/core/tsi/ssl_transport_security.h
+++ b/src/core/tsi/ssl_transport_security.h
@@ -36,6 +36,20 @@
 
 #define TSI_SSL_ALPN_SELECTED_PROTOCOL "ssl_alpn_selected_protocol"
 
+/* --- tsi_ssl_root_certs_store object ---
+
+   This object stores SSL root certificates. It can be shared by multiple SSL
+   context. */
+typedef struct tsi_ssl_root_certs_store tsi_ssl_root_certs_store;
+
+/* Given a NULL-terminated string containing the PEM encoding of the root
+   certificates, creates a tsi_ssl_root_certs_store object. */
+tsi_ssl_root_certs_store* tsi_ssl_root_certs_store_create(
+    const char* pem_roots);
+
+/* Destroys the tsi_ssl_root_certs_store object. */
+void tsi_ssl_root_certs_store_destroy(tsi_ssl_root_certs_store* self);
+
 /* --- tsi_ssl_session_cache object ---
 
    Cache for SSL sessions for sessions resumption.  */
@@ -70,13 +84,13 @@ typedef struct {
   const char* cert_chain;
 } tsi_ssl_pem_key_cert_pair;
 
-/* Creates a client handshaker factory.
+/* TO BE DEPRECATED.
+   Creates a client handshaker factory.
    - pem_key_cert_pair is a pointer to the object containing client's private
      key and certificate chain. This parameter can be NULL if the client does
      not have such a key/cert pair.
    - pem_roots_cert is the NULL-terminated string containing the PEM encoding of
-     the client root certificates. This parameter may be NULL if the server does
-     not want the client to be authenticated with SSL.
+     the server root certificates.
    - cipher_suites contains an optional list of the ciphers that the client
      supports. The format of this string is described in:
      https://www.openssl.org/docs/apps/ciphers.html.
@@ -103,9 +117,13 @@ typedef struct {
      not have such a key/cert pair. */
   const tsi_ssl_pem_key_cert_pair* pem_key_cert_pair;
   /* pem_roots_cert is the NULL-terminated string containing the PEM encoding of
-     the client root certificates. This parameter may be NULL if the server does
-     not want the client to be authenticated with SSL. */
+     the client root certificates. */
   const char* pem_root_certs;
+  /* root_store is a pointer to the ssl_root_certs_store object. If root_store
+    is not nullptr and SSL implementation permits, root_store will be used as
+    root certificates. Otherwise, pem_roots_cert will be used to load server
+    root certificates. */
+  const tsi_ssl_root_certs_store* root_store;
   /* cipher_suites contains an optional list of the ciphers that the client
      supports. The format of this string is described in:
      https://www.openssl.org/docs/apps/ciphers.html.
@@ -160,12 +178,14 @@ void tsi_ssl_client_handshaker_factory_unref(
 typedef struct tsi_ssl_server_handshaker_factory
     tsi_ssl_server_handshaker_factory;
 
-/* Creates a server handshaker factory.
+/* TO BE DEPRECATED.
+   Creates a server handshaker factory.
    - pem_key_cert_pairs is an array private key / certificate chains of the
      server.
    - num_key_cert_pairs is the number of items in the pem_key_cert_pairs array.
    - pem_root_certs is the NULL-terminated string containing the PEM encoding
-     of the server root certificates.
+     of the client root certificates. This parameter may be NULL if the server
+     does not want the client to be authenticated with SSL.
    - cipher_suites contains an optional list of the ciphers that the server
      supports. The format of this string is described in:
      https://www.openssl.org/docs/apps/ciphers.html.
@@ -187,7 +207,8 @@ tsi_result tsi_create_ssl_server_handshaker_factory(
     const char** alpn_protocols, uint16_t num_alpn_protocols,
     tsi_ssl_server_handshaker_factory** factory);
 
-/* Same as tsi_create_ssl_server_handshaker_factory method except uses
+/* TO BE DEPRECATED.
+   Same as tsi_create_ssl_server_handshaker_factory method except uses
    tsi_client_certificate_request_type to support more ways to handle client
    certificate authentication.
    - client_certificate_request, if set to non-zero will force the client to
@@ -208,7 +229,8 @@ typedef struct {
      array. */
   size_t num_key_cert_pairs;
   /* pem_root_certs is the NULL-terminated string containing the PEM encoding
-     of the server root certificates. */
+     of the server root certificates. This parameter may be NULL if the server
+     does not want the client to be authenticated with SSL. */
   const char* pem_client_root_certs;
   /* client_certificate_request, if set to non-zero will force the client to
      authenticate with an SSL cert. Note that this option is ignored if