Merge branch 'master' into rq-threads-2

40 files changed, 611 insertions, 134 deletions

diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index 9ad271753c..bc6fa0d0eb 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -813,11 +813,7 @@ static void set_write_state(grpc_chttp2_transport* t,
                                  write_state_name(st), reason));
   t->write_state = st;
   if (st == GRPC_CHTTP2_WRITE_STATE_IDLE) {
-    grpc_chttp2_stream* s;
-    while (grpc_chttp2_list_pop_waiting_for_write_stream(t, &s)) {
-      GRPC_CLOSURE_LIST_SCHED(&s->run_after_write);
-      GRPC_CHTTP2_STREAM_UNREF(s, "chttp2:write_closure_sched");
-    }
+    GRPC_CLOSURE_LIST_SCHED(&t->run_after_write);
     if (t->close_transport_on_writes_finished != nullptr) {
       grpc_error* err = t->close_transport_on_writes_finished;
       t->close_transport_on_writes_finished = nullptr;
@@ -1212,10 +1208,7 @@ void grpc_chttp2_complete_closure_step(grpc_chttp2_transport* t,
         !(closure->next_data.scratch & CLOSURE_BARRIER_MAY_COVER_WRITE)) {
       GRPC_CLOSURE_RUN(closure, closure->error_data.error);
     } else {
-      if (grpc_chttp2_list_add_waiting_for_write_stream(t, s)) {
-        GRPC_CHTTP2_STREAM_REF(s, "chttp2:pending_write_closure");
-      }
-      grpc_closure_list_append(&s->run_after_write, closure,
+      grpc_closure_list_append(&t->run_after_write, closure,
                                closure->error_data.error);
     }
   }
@@ -2016,10 +2009,6 @@ static void remove_stream(grpc_chttp2_transport* t, uint32_t id,
 
 void grpc_chttp2_cancel_stream(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
                                grpc_error* due_to_error) {
-  GRPC_CLOSURE_LIST_SCHED(&s->run_after_write);
-  if (grpc_chttp2_list_remove_waiting_for_write_stream(t, s)) {
-    GRPC_CHTTP2_STREAM_UNREF(s, "chttp2:pending_write_closure");
-  }
   if (!t->is_client && !s->sent_trailing_metadata &&
       grpc_error_has_clear_grpc_status(due_to_error)) {
     close_from_api(t, s, due_to_error);
diff --git a/src/core/ext/transport/chttp2/transport/flow_control.cc b/src/core/ext/transport/chttp2/transport/flow_control.cc
index e89c363200..5f3dd98461 100644
--- a/src/core/ext/transport/chttp2/transport/flow_control.cc
+++ b/src/core/ext/transport/chttp2/transport/flow_control.cc
@@ -55,7 +55,7 @@ static char* fmt_int64_diff_str(int64_t old_val, int64_t new_val) {
 
 static char* fmt_uint32_diff_str(uint32_t old_val, uint32_t new_val) {
   char* str;
-  if (new_val > 0 && old_val != new_val) {
+  if (old_val != new_val) {
     gpr_asprintf(&str, "%" PRIu32 " -> %" PRIu32 "", old_val, new_val);
   } else {
     gpr_asprintf(&str, "%" PRIu32 "", old_val);
@@ -98,10 +98,12 @@ void FlowControlTrace::Finish() {
   if (sfc_ != nullptr) {
     srw_str = fmt_int64_diff_str(remote_window_delta_ + remote_window,
                                  sfc_->remote_window_delta() + remote_window);
-    slw_str = fmt_int64_diff_str(local_window_delta_ + acked_local_window,
-                                 local_window_delta_ + acked_local_window);
-    saw_str = fmt_int64_diff_str(announced_window_delta_ + acked_local_window,
-                                 announced_window_delta_ + acked_local_window);
+    slw_str =
+        fmt_int64_diff_str(local_window_delta_ + acked_local_window,
+                           sfc_->local_window_delta() + acked_local_window);
+    saw_str =
+        fmt_int64_diff_str(announced_window_delta_ + acked_local_window,
+                           sfc_->announced_window_delta() + acked_local_window);
   } else {
     srw_str = gpr_leftpad("", ' ', kTracePadding);
     slw_str = gpr_leftpad("", ' ', kTracePadding);
diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h
index 4f1a08d98b..ca6e715978 100644
--- a/src/core/ext/transport/chttp2/transport/internal.h
+++ b/src/core/ext/transport/chttp2/transport/internal.h
@@ -54,8 +54,6 @@ typedef enum {
   /** streams that are waiting to start because there are too many concurrent
       streams on the connection */
   GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY,
-  /** streams with closures waiting to be run on a write **/
-  GRPC_CHTTP2_LIST_WAITING_FOR_WRITE,
   STREAM_LIST_COUNT /* must be last */
 } grpc_chttp2_stream_list_id;
 
@@ -433,6 +431,9 @@ struct grpc_chttp2_transport {
    */
   grpc_error* close_transport_on_writes_finished;
 
+  /* a list of closures to run after writes are finished */
+  grpc_closure_list run_after_write;
+
   /* buffer pool state */
   /** have we scheduled a benign cleanup? */
   bool benign_reclaimer_registered;
@@ -583,7 +584,6 @@ struct grpc_chttp2_stream {
 
   grpc_slice_buffer flow_controlled_buffer;
 
-  grpc_closure_list run_after_write;
   grpc_chttp2_write_cb* on_flow_controlled_cbs;
   grpc_chttp2_write_cb* on_write_finished_cbs;
   grpc_chttp2_write_cb* finish_after_write;
@@ -686,13 +686,6 @@ bool grpc_chttp2_list_pop_stalled_by_stream(grpc_chttp2_transport* t,
 bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport* t,
                                                grpc_chttp2_stream* s);
 
-bool grpc_chttp2_list_add_waiting_for_write_stream(grpc_chttp2_transport* t,
-                                                   grpc_chttp2_stream* s);
-bool grpc_chttp2_list_pop_waiting_for_write_stream(grpc_chttp2_transport* t,
-                                                   grpc_chttp2_stream** s);
-bool grpc_chttp2_list_remove_waiting_for_write_stream(grpc_chttp2_transport* t,
-                                                      grpc_chttp2_stream* s);
-
 /********* Flow Control ***************/
 
 // Takes in a flow control action and performs all the needed operations.
diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.cc b/src/core/ext/transport/chttp2/transport/stream_lists.cc
index 50bfe36a86..6626170a7e 100644
--- a/src/core/ext/transport/chttp2/transport/stream_lists.cc
+++ b/src/core/ext/transport/chttp2/transport/stream_lists.cc
@@ -35,8 +35,6 @@ static const char* stream_list_id_string(grpc_chttp2_stream_list_id id) {
       return "stalled_by_stream";
     case GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY:
       return "waiting_for_concurrency";
-    case GRPC_CHTTP2_LIST_WAITING_FOR_WRITE:
-      return "waiting_for_write";
     case STREAM_LIST_COUNT:
       GPR_UNREACHABLE_CODE(return "unknown");
   }
@@ -216,18 +214,3 @@ bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport* t,
                                                grpc_chttp2_stream* s) {
   return stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM);
 }
-
-bool grpc_chttp2_list_add_waiting_for_write_stream(grpc_chttp2_transport* t,
-                                                   grpc_chttp2_stream* s) {
-  return stream_list_add(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_WRITE);
-}
-
-bool grpc_chttp2_list_pop_waiting_for_write_stream(grpc_chttp2_transport* t,
-                                                   grpc_chttp2_stream** s) {
-  return stream_list_pop(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_WRITE);
-}
-
-bool grpc_chttp2_list_remove_waiting_for_write_stream(grpc_chttp2_transport* t,
-                                                      grpc_chttp2_stream* s) {
-  return stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_WRITE);
-}
diff --git a/src/core/lib/iomgr/call_combiner.h b/src/core/lib/iomgr/call_combiner.h
index 641fa18082..6f7ddd4043 100644
--- a/src/core/lib/iomgr/call_combiner.h
+++ b/src/core/lib/iomgr/call_combiner.h
@@ -102,7 +102,10 @@ void grpc_call_combiner_stop(grpc_call_combiner* call_combiner,
 /// If \a closure is NULL, then no closure will be invoked on
 /// cancellation; this effectively unregisters the previously set closure.
 /// However, most filters will not need to explicitly unregister their
-/// callbacks, as this is done automatically when the call is destroyed.
+/// callbacks, as this is done automatically when the call is destroyed. Filters
+/// that schedule the cancellation closure on ExecCtx do not need to take a ref
+/// on the call stack to guarantee closure liveness. This is done by explicitly
+/// flushing ExecCtx after the unregistration during call destruction.
 void grpc_call_combiner_set_notify_on_cancel(grpc_call_combiner* call_combiner,
                                              grpc_closure* closure);
 
diff --git a/src/core/lib/iomgr/iomgr_posix_cfstream.cc b/src/core/lib/iomgr/iomgr_posix_cfstream.cc
new file mode 100644
index 0000000000..235a9e0712
--- /dev/null
+++ b/src/core/lib/iomgr/iomgr_posix_cfstream.cc
@@ -0,0 +1,75 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/iomgr/port.h"
+
+#ifdef GRPC_CFSTREAM_IOMGR
+
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/iomgr/ev_posix.h"
+#include "src/core/lib/iomgr/iomgr_internal.h"
+#include "src/core/lib/iomgr/iomgr_posix.h"
+#include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/tcp_client.h"
+#include "src/core/lib/iomgr/tcp_posix.h"
+#include "src/core/lib/iomgr/tcp_server.h"
+#include "src/core/lib/iomgr/timer.h"
+
+static const char* grpc_cfstream_env_var = "grpc_cfstream";
+
+extern grpc_tcp_server_vtable grpc_posix_tcp_server_vtable;
+extern grpc_tcp_client_vtable grpc_posix_tcp_client_vtable;
+extern grpc_tcp_client_vtable grpc_cfstream_client_vtable;
+extern grpc_timer_vtable grpc_generic_timer_vtable;
+extern grpc_pollset_vtable grpc_posix_pollset_vtable;
+extern grpc_pollset_set_vtable grpc_posix_pollset_set_vtable;
+extern grpc_address_resolver_vtable grpc_posix_resolver_vtable;
+
+static void iomgr_platform_init(void) {
+  grpc_wakeup_fd_global_init();
+  grpc_event_engine_init();
+}
+
+static void iomgr_platform_flush(void) {}
+
+static void iomgr_platform_shutdown(void) {
+  grpc_event_engine_shutdown();
+  grpc_wakeup_fd_global_destroy();
+}
+
+static grpc_iomgr_platform_vtable vtable = {
+    iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown};
+
+void grpc_set_default_iomgr_platform() {
+  char* enable_cfstream = getenv(grpc_cfstream_env_var);
+  grpc_tcp_client_vtable* client_vtable = &grpc_posix_tcp_client_vtable;
+  if (enable_cfstream != nullptr && enable_cfstream[0] == '1') {
+    client_vtable = &grpc_cfstream_client_vtable;
+  }
+  grpc_set_tcp_client_impl(client_vtable);
+  grpc_set_tcp_server_impl(&grpc_posix_tcp_server_vtable);
+  grpc_set_timer_impl(&grpc_generic_timer_vtable);
+  grpc_set_pollset_vtable(&grpc_posix_pollset_vtable);
+  grpc_set_pollset_set_vtable(&grpc_posix_pollset_set_vtable);
+  grpc_set_resolver_impl(&grpc_posix_resolver_vtable);
+  grpc_set_iomgr_platform_vtable(&vtable);
+}
+
+#endif /* GRPC_CFSTREAM_IOMGR */
diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h
index 80d8e63cdd..1d0ecff802 100644
--- a/src/core/lib/iomgr/port.h
+++ b/src/core/lib/iomgr/port.h
@@ -98,9 +98,9 @@
 #define GRPC_POSIX_FORK 1
 #define GRPC_POSIX_NO_SPECIAL_WAKEUP_FD 1
 #ifdef GRPC_CFSTREAM
-#define GRPC_POSIX_SOCKET_IOMGR 1
-#define GRPC_CFSTREAM_ENDPOINT 1
+#define GRPC_CFSTREAM_IOMGR 1
 #define GRPC_CFSTREAM_CLIENT 1
+#define GRPC_CFSTREAM_ENDPOINT 1
 #define GRPC_POSIX_SOCKET_ARES_EV_DRIVER 1
 #define GRPC_POSIX_SOCKET_EV 1
 #define GRPC_POSIX_SOCKET_EV_EPOLL1 1
@@ -111,6 +111,7 @@
 #define GRPC_POSIX_SOCKET_SOCKADDR 1
 #define GRPC_POSIX_SOCKET_SOCKET_FACTORY 1
 #define GRPC_POSIX_SOCKET_TCP 1
+#define GRPC_POSIX_SOCKET_TCP_CLIENT 1
 #define GRPC_POSIX_SOCKET_TCP_SERVER 1
 #define GRPC_POSIX_SOCKET_TCP_SERVER_UTILS_COMMON 1
 #define GRPC_POSIX_SOCKET_UTILS_COMMON 1
diff --git a/src/core/lib/iomgr/tcp_client_cfstream.cc b/src/core/lib/iomgr/tcp_client_cfstream.cc
index 5acea91792..4b21322d74 100644
--- a/src/core/lib/iomgr/tcp_client_cfstream.cc
+++ b/src/core/lib/iomgr/tcp_client_cfstream.cc
@@ -211,6 +211,6 @@ static void CFStreamClientConnect(grpc_closure* closure, grpc_endpoint** ep,
   gpr_mu_unlock(&connect->mu);
 }
 
-grpc_tcp_client_vtable grpc_posix_tcp_client_vtable = {CFStreamClientConnect};
+grpc_tcp_client_vtable grpc_cfstream_client_vtable = {CFStreamClientConnect};
 
 #endif /* GRPC_CFSTREAM_CLIENT */
diff --git a/src/core/lib/security/security_connector/load_system_roots.h b/src/core/lib/security/security_connector/load_system_roots.h
new file mode 100644
index 0000000000..5fdec15498
--- /dev/null
+++ b/src/core/lib/security/security_connector/load_system_roots.h
@@ -0,0 +1,29 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H
+#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H
+
+namespace grpc_core {
+
+// Returns a slice containing roots from the OS trust store
+grpc_slice LoadSystemRootCerts();
+
+}  // namespace grpc_core
+
+#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H */
diff --git a/src/core/lib/security/security_connector/load_system_roots_fallback.cc b/src/core/lib/security/security_connector/load_system_roots_fallback.cc
new file mode 100644
index 0000000000..73d1245f33
--- /dev/null
+++ b/src/core/lib/security/security_connector/load_system_roots_fallback.cc
@@ -0,0 +1,32 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include <grpc/slice_buffer.h>
+#include "src/core/lib/security/security_connector/load_system_roots.h"
+
+#ifndef GPR_LINUX
+
+namespace grpc_core {
+
+grpc_slice LoadSystemRootCerts() { return grpc_empty_slice(); }
+
+}  // namespace grpc_core
+
+#endif /* GPR_LINUX */
diff --git a/src/core/lib/security/security_connector/load_system_roots_linux.cc b/src/core/lib/security/security_connector/load_system_roots_linux.cc
new file mode 100644
index 0000000000..924fa8a3e2
--- /dev/null
+++ b/src/core/lib/security/security_connector/load_system_roots_linux.cc
@@ -0,0 +1,165 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include <grpc/slice_buffer.h>
+#include "src/core/lib/security/security_connector/load_system_roots_linux.h"
+
+#ifdef GPR_LINUX
+
+#include "src/core/lib/security/security_connector/load_system_roots.h"
+
+#include <dirent.h>
+#include <fcntl.h>
+#include <stdbool.h>
+#include <string.h>
+#include <sys/param.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+#include "src/core/lib/gpr/env.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/useful.h"
+#include "src/core/lib/gprpp/inlined_vector.h"
+#include "src/core/lib/iomgr/load_file.h"
+
+namespace grpc_core {
+namespace {
+
+const char* kLinuxCertFiles[] = {
+    "/etc/ssl/certs/ca-certificates.crt", "/etc/pki/tls/certs/ca-bundle.crt",
+    "/etc/ssl/ca-bundle.pem", "/etc/pki/tls/cacert.pem",
+    "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"};
+const char* kLinuxCertDirectories[] = {
+    "/etc/ssl/certs", "/system/etc/security/cacerts", "/usr/local/share/certs",
+    "/etc/pki/tls/certs", "/etc/openssl/certs"};
+
+grpc_slice GetSystemRootCerts() {
+  grpc_slice valid_bundle_slice = grpc_empty_slice();
+  size_t num_cert_files_ = GPR_ARRAY_SIZE(kLinuxCertFiles);
+  for (size_t i = 0; i < num_cert_files_; i++) {
+    grpc_error* error =
+        grpc_load_file(kLinuxCertFiles[i], 1, &valid_bundle_slice);
+    if (error == GRPC_ERROR_NONE) {
+      return valid_bundle_slice;
+    }
+  }
+  return grpc_empty_slice();
+}
+
+}  // namespace
+
+void GetAbsoluteFilePath(const char* valid_file_dir,
+                         const char* file_entry_name, char* path_buffer) {
+  if (valid_file_dir != nullptr && file_entry_name != nullptr) {
+    int path_len = snprintf(path_buffer, MAXPATHLEN, "%s/%s", valid_file_dir,
+                            file_entry_name);
+    if (path_len == 0) {
+      gpr_log(GPR_ERROR, "failed to get absolute path for file: %s",
+              file_entry_name);
+    }
+  }
+}
+
+grpc_slice CreateRootCertsBundle(const char* certs_directory) {
+  grpc_slice bundle_slice = grpc_empty_slice();
+  if (certs_directory == nullptr) {
+    return bundle_slice;
+  }
+  DIR* ca_directory = opendir(certs_directory);
+  if (ca_directory == nullptr) {
+    return bundle_slice;
+  }
+  struct FileData {
+    char path[MAXPATHLEN];
+    off_t size;
+  };
+  InlinedVector<FileData, 2> roots_filenames;
+  size_t total_bundle_size = 0;
+  struct dirent* directory_entry;
+  while ((directory_entry = readdir(ca_directory)) != nullptr) {
+    struct stat dir_entry_stat;
+    const char* file_entry_name = directory_entry->d_name;
+    FileData file_data;
+    GetAbsoluteFilePath(certs_directory, file_entry_name, file_data.path);
+    int stat_return = stat(file_data.path, &dir_entry_stat);
+    if (stat_return == -1 || !S_ISREG(dir_entry_stat.st_mode)) {
+      // no subdirectories.
+      if (stat_return == -1) {
+        gpr_log(GPR_ERROR, "failed to get status for file: %s", file_data.path);
+      }
+      continue;
+    }
+    file_data.size = dir_entry_stat.st_size;
+    total_bundle_size += file_data.size;
+    roots_filenames.push_back(file_data);
+  }
+  closedir(ca_directory);
+  char* bundle_string = static_cast<char*>(gpr_zalloc(total_bundle_size + 1));
+  size_t bytes_read = 0;
+  for (size_t i = 0; i < roots_filenames.size(); i++) {
+    int file_descriptor = open(roots_filenames[i].path, O_RDONLY);
+    if (file_descriptor != -1) {
+      // Read file into bundle.
+      size_t cert_file_size = roots_filenames[i].size;
+      int read_ret =
+          read(file_descriptor, bundle_string + bytes_read, cert_file_size);
+      if (read_ret != -1) {
+        bytes_read += read_ret;
+      } else {
+        gpr_log(GPR_ERROR, "failed to read file: %s", roots_filenames[i].path);
+      }
+    }
+  }
+  bundle_slice = grpc_slice_new(bundle_string, bytes_read, gpr_free);
+  return bundle_slice;
+}
+
+grpc_slice LoadSystemRootCerts() {
+  grpc_slice result = grpc_empty_slice();
+  // Prioritize user-specified custom directory if flag is set.
+  char* custom_dir = gpr_getenv("GRPC_SYSTEM_SSL_ROOTS_DIR");
+  if (custom_dir != nullptr) {
+    result = CreateRootCertsBundle(custom_dir);
+    gpr_free(custom_dir);
+  }
+  // If the custom directory is empty/invalid/not specified, fallback to
+  // distribution-specific directory.
+  if (GRPC_SLICE_IS_EMPTY(result)) {
+    result = GetSystemRootCerts();
+  }
+  if (GRPC_SLICE_IS_EMPTY(result)) {
+    for (size_t i = 0; i < GPR_ARRAY_SIZE(kLinuxCertDirectories); i++) {
+      result = CreateRootCertsBundle(kLinuxCertDirectories[i]);
+      if (!GRPC_SLICE_IS_EMPTY(result)) {
+        break;
+      }
+    }
+  }
+  return result;
+}
+
+}  // namespace grpc_core
+
+#endif /* GPR_LINUX */
diff --git a/src/core/lib/security/security_connector/load_system_roots_linux.h b/src/core/lib/security/security_connector/load_system_roots_linux.h
new file mode 100644
index 0000000000..12617df492
--- /dev/null
+++ b/src/core/lib/security/security_connector/load_system_roots_linux.h
@@ -0,0 +1,44 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H
+#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H
+
+#include <grpc/support/port_platform.h>
+
+#ifdef GPR_LINUX
+
+namespace grpc_core {
+
+// Creates a bundle slice containing the contents of all certificate files in
+// a directory.
+// Returns such slice.
+// Exposed for testing purposes only.
+grpc_slice CreateRootCertsBundle(const char* certs_directory);
+
+// Gets the absolute file path needed to load a certificate file.
+// Populates path_buffer, which must be of size MAXPATHLEN.
+// Exposed for testing purposes only.
+void GetAbsoluteFilePath(const char* valid_file_dir,
+                         const char* file_entry_name, char* path_buffer);
+
+}  // namespace grpc_core
+
+#endif /* GPR_LINUX */
+#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H \
+        */
diff --git a/src/core/lib/security/security_connector/security_connector.cc b/src/core/lib/security/security_connector/security_connector.cc
index 59cf3a0af1..04b4c87c71 100644
--- a/src/core/lib/security/security_connector/security_connector.cc
+++ b/src/core/lib/security/security_connector/security_connector.cc
@@ -21,7 +21,6 @@
 #include "src/core/lib/security/security_connector/security_connector.h"
 
 #include <stdbool.h>
-#include <string.h>
 
 #include <grpc/slice_buffer.h>
 #include <grpc/support/alloc.h>
@@ -39,6 +38,7 @@
 #include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/security/credentials/fake/fake_credentials.h"
 #include "src/core/lib/security/credentials/ssl/ssl_credentials.h"
+#include "src/core/lib/security/security_connector/load_system_roots.h"
 #include "src/core/lib/security/transport/secure_endpoint.h"
 #include "src/core/lib/security/transport/security_handshaker.h"
 #include "src/core/lib/security/transport/target_authority_table.h"
@@ -57,6 +57,12 @@ static const char* installed_roots_path =
     INSTALL_PREFIX "/share/grpc/roots.pem";
 #endif
 
+/** Environment variable used as a flag to enable/disable loading system root
+    certificates from the OS trust store. */
+#ifndef GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR
+#define GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR "GRPC_USE_SYSTEM_SSL_ROOTS"
+#endif
+
 #ifndef TSI_OPENSSL_ALPN_SUPPORT
 #define TSI_OPENSSL_ALPN_SUPPORT 1
 #endif
@@ -1186,6 +1192,10 @@ const char* DefaultSslRootStore::GetPemRootCerts() {
 
 grpc_slice DefaultSslRootStore::ComputePemRootCerts() {
   grpc_slice result = grpc_empty_slice();
+  char* use_system_roots_env_value =
+      gpr_getenv(GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR);
+  const bool use_system_roots = gpr_is_true(use_system_roots_env_value);
+  gpr_free(use_system_roots_env_value);
   // First try to load the roots from the environment.
   char* default_root_certs_path =
       gpr_getenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR);
@@ -1207,7 +1217,11 @@ grpc_slice DefaultSslRootStore::ComputePemRootCerts() {
     }
     gpr_free(pem_root_certs);
   }
-  // Fall back to installed certs if needed.
+  // Try loading roots from OS trust store if flag is enabled.
+  if (GRPC_SLICE_IS_EMPTY(result) && use_system_roots) {
+    result = LoadSystemRootCerts();
+  }
+  // Fallback to roots manually shipped with gRPC.
   if (GRPC_SLICE_IS_EMPTY(result) &&
       ovrd_res != GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY) {
     GRPC_LOG_IF_ERROR("load_file",
diff --git a/src/core/lib/security/transport/client_auth_filter.cc b/src/core/lib/security/transport/client_auth_filter.cc
index 9b5c6f3490..0f125e7c26 100644
--- a/src/core/lib/security/transport/client_auth_filter.cc
+++ b/src/core/lib/security/transport/client_auth_filter.cc
@@ -167,7 +167,6 @@ static void cancel_get_request_metadata(void* arg, grpc_error* error) {
     grpc_call_credentials_cancel_get_request_metadata(
         calld->creds, &calld->md_array, GRPC_ERROR_REF(error));
   }
-  GRPC_CALL_STACK_UNREF(calld->owning_call, "cancel_get_request_metadata");
 }
 
 static void send_security_metadata(grpc_call_element* elem,
@@ -222,7 +221,6 @@ static void send_security_metadata(grpc_call_element* elem,
     GRPC_ERROR_UNREF(error);
   } else {
     // Async return; register cancellation closure with call combiner.
-    GRPC_CALL_STACK_REF(calld->owning_call, "cancel_get_request_metadata");
     grpc_call_combiner_set_notify_on_cancel(
         calld->call_combiner,
         GRPC_CLOSURE_INIT(&calld->get_request_metadata_cancel_closure,
@@ -265,7 +263,6 @@ static void cancel_check_call_host(void* arg, grpc_error* error) {
         chand->security_connector, &calld->async_result_closure,
         GRPC_ERROR_REF(error));
   }
-  GRPC_CALL_STACK_UNREF(calld->owning_call, "cancel_check_call_host");
 }
 
 static void auth_start_transport_stream_op_batch(
@@ -318,7 +315,6 @@ static void auth_start_transport_stream_op_batch(
         GRPC_ERROR_UNREF(error);
       } else {
         // Async return; register cancellation closure with call combiner.
-        GRPC_CALL_STACK_REF(calld->owning_call, "cancel_check_call_host");
         grpc_call_combiner_set_notify_on_cancel(
             calld->call_combiner,
             GRPC_CLOSURE_INIT(&calld->check_call_host_cancel_closure,
diff --git a/src/core/lib/security/transport/server_auth_filter.cc b/src/core/lib/security/transport/server_auth_filter.cc
index 2dbefdf131..19cbb03b63 100644
--- a/src/core/lib/security/transport/server_auth_filter.cc
+++ b/src/core/lib/security/transport/server_auth_filter.cc
@@ -156,7 +156,6 @@ static void cancel_call(void* arg, grpc_error* error) {
     on_md_processing_done_inner(elem, nullptr, 0, nullptr, 0,
                                 GRPC_ERROR_REF(error));
   }
-  GRPC_CALL_STACK_UNREF(calld->owning_call, "cancel_call");
 }
 
 static void recv_initial_metadata_ready(void* arg, grpc_error* error) {
@@ -168,7 +167,6 @@ static void recv_initial_metadata_ready(void* arg, grpc_error* error) {
     if (chand->creds != nullptr && chand->creds->processor.process != nullptr) {
       // We're calling out to the application, so we need to make sure
       // to drop the call combiner early if we get cancelled.
-      GRPC_CALL_STACK_REF(calld->owning_call, "cancel_call");
       GRPC_CLOSURE_INIT(&calld->cancel_closure, cancel_call, elem,
                         grpc_schedule_on_exec_ctx);
       grpc_call_combiner_set_notify_on_cancel(calld->call_combiner,
diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc
index dbad5ded4d..52053e686b 100644
--- a/src/core/lib/surface/call.cc
+++ b/src/core/lib/surface/call.cc
@@ -613,8 +613,11 @@ void grpc_call_unref(grpc_call* c) {
     // Unset the call combiner cancellation closure.  This has the
     // effect of scheduling the previously set cancellation closure, if
     // any, so that it can release any internal references it may be
-    // holding to the call stack.
+    // holding to the call stack. Also flush the closures on exec_ctx so that
+    // filters that schedule cancel notification closures on exec_ctx do not
+    // need to take a ref of the call stack to guarantee closure liveness.
     grpc_call_combiner_set_notify_on_cancel(&c->call_combiner, nullptr);
+    grpc_core::ExecCtx::Get()->Flush();
   }
   GRPC_CALL_INTERNAL_UNREF(c, "destroy");
 }
diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj
index 18993a93e0..d58f046824 100755
--- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj
+++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj
@@ -17,8 +17,8 @@
 
   <ItemGroup>
     <PackageReference Include="Newtonsoft.Json" Version="9.0.1" />
-    <PackageReference Include="NUnit" Version="3.6.0" />
-    <PackageReference Include="NUnitLite" Version="3.6.0" />
+    <PackageReference Include="NUnit" Version="3.10.1" />
+    <PackageReference Include="NUnitLite" Version="3.10.1" />
     <PackageReference Include="OpenCover" Version="4.6.519" />
     <PackageReference Include="ReportGenerator" Version="2.4.4.0" />
   </ItemGroup>
diff --git a/src/csharp/Grpc.Core.Tests/SanityTest.cs b/src/csharp/Grpc.Core.Tests/SanityTest.cs
index 73efad1f84..eaad409ec0 100644
--- a/src/csharp/Grpc.Core.Tests/SanityTest.cs
+++ b/src/csharp/Grpc.Core.Tests/SanityTest.cs
@@ -65,13 +65,13 @@ namespace Grpc.Core.Tests
                 {
                     foreach (var m in t.GetMethods())
                     {
-                        var attributes = m.GetCustomAttributes(typeof(NUnit.Framework.TestAttribute), true);
-                        if (attributes.Length > 0)
+                        var testAttributes = m.GetCustomAttributes(typeof(NUnit.Framework.TestAttribute), true);
+                        var testCaseAttributes = m.GetCustomAttributes(typeof(NUnit.Framework.TestCaseAttribute), true);
+                        if (testAttributes.Length > 0 || testCaseAttributes.Length > 0)
                         {
                             testClasses.Add(t.FullName);
                             break;
                         }
-
                     }
                 }
                 testClasses.Sort();
diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj
index fc32271063..dc5683c975 100755
--- a/src/csharp/Grpc.Core/Grpc.Core.csproj
+++ b/src/csharp/Grpc.Core/Grpc.Core.csproj
@@ -47,35 +47,35 @@
       <Pack>true</Pack>
     </Content>
     <Content Include="..\nativelibs\csharp_ext_linux_android_armeabi-v7a\libgrpc_csharp_ext.so">
-      <PackagePath>runtimes/monoandroid/armeabi-v7a/libgrpc_csharp_ext.so</PackagePath>
+      <PackagePath>native/android/armeabi-v7a/libgrpc_csharp_ext.so</PackagePath>
       <Pack>true</Pack>
     </Content>
     <Content Include="..\nativelibs\csharp_ext_linux_android_arm64-v8a\libgrpc_csharp_ext.so">
-      <PackagePath>runtimes/monoandroid/arm64-v8a/libgrpc_csharp_ext.so</PackagePath>
+      <PackagePath>native/android/arm64-v8a/libgrpc_csharp_ext.so</PackagePath>
       <Pack>true</Pack>
     </Content>
     <Content Include="..\nativelibs\csharp_ext_linux_android_x86\libgrpc_csharp_ext.so">
-      <PackagePath>runtimes/monoandroid/x86/libgrpc_csharp_ext.so</PackagePath>
+      <PackagePath>native/android/x86/libgrpc_csharp_ext.so</PackagePath>
       <Pack>true</Pack>
     </Content>
     <Content Include="..\nativelibs\csharp_ext_macos_ios\libgrpc_csharp_ext.a">
-      <PackagePath>runtimes/ios/native/libgrpc_csharp_ext.a</PackagePath>
+      <PackagePath>native/ios/universal/libgrpc_csharp_ext.a</PackagePath>
       <Pack>true</Pack>
     </Content>
     <Content Include="..\nativelibs\csharp_ext_macos_ios\libgrpc.a">
-      <PackagePath>runtimes/ios/native/libgrpc.a</PackagePath>
+      <PackagePath>native/ios/universal/libgrpc.a</PackagePath>
       <Pack>true</Pack>
     </Content>
     <Content Include="build\net45\Grpc.Core.targets">
       <PackagePath>build/net45/</PackagePath>
       <Pack>true</Pack>
     </Content>
-    <Content Include="build\MonoAndroid\Grpc.Core.targets">
-      <PackagePath>build/MonoAndroid/</PackagePath>
+    <Content Include="build\MonoAndroid10\Grpc.Core.targets">
+      <PackagePath>build/MonoAndroid10/</PackagePath>
       <Pack>true</Pack>
     </Content>
-    <Content Include="build\Xamarin.iOS\Grpc.Core.targets">
-      <PackagePath>build/Xamarin.iOS/</PackagePath>
+    <Content Include="build\Xamarin.iOS10\Grpc.Core.targets">
+      <PackagePath>build/Xamarin.iOS10/</PackagePath>
       <Pack>true</Pack>
     </Content>
   </ItemGroup>
diff --git a/src/csharp/Grpc.Core/GrpcEnvironment.cs b/src/csharp/Grpc.Core/GrpcEnvironment.cs
index a6a1d8af50..6ca694e0e4 100644
--- a/src/csharp/Grpc.Core/GrpcEnvironment.cs
+++ b/src/csharp/Grpc.Core/GrpcEnvironment.cs
@@ -50,6 +50,7 @@ namespace Grpc.Core
         static int requestCallContextPoolThreadLocalCapacity = DefaultRequestCallContextPoolThreadLocalCapacity;
         static readonly HashSet<Channel> registeredChannels = new HashSet<Channel>();
         static readonly HashSet<Server> registeredServers = new HashSet<Server>();
+        static readonly AtomicCounter nativeInitCounter = new AtomicCounter();
 
         static ILogger logger = new LogLevelFilterLogger(new ConsoleLogger(), LogLevel.Off, true);
 
@@ -360,12 +361,25 @@ namespace Grpc.Core
 
         internal static void GrpcNativeInit()
         {
+            if (!IsNativeShutdownAllowed && nativeInitCounter.Count > 0)
+            {
+                // Normally grpc_init and grpc_shutdown calls should come in pairs (C core does reference counting),
+                // but in case we avoid grpc_shutdown calls altogether, calling grpc_init has no effect
+                // besides incrementing an internal C core counter that could theoretically overflow.
+                // To avoid this theoretical possibility we guard repeated calls to grpc_init()
+                // with a 64-bit atomic counter (that can't realistically overflow).
+                return;
+            }
             NativeMethods.Get().grpcsharp_init();
+            nativeInitCounter.Increment();
         }
 
         internal static void GrpcNativeShutdown()
         {
-            NativeMethods.Get().grpcsharp_shutdown();
+            if (IsNativeShutdownAllowed)
+            {
+                NativeMethods.Get().grpcsharp_shutdown();
+            }
         }
 
         /// <summary>
@@ -411,6 +425,14 @@ namespace Grpc.Core
             return GetThreadPoolSizeOrDefault();
         }
 
+        // On some platforms (specifically iOS), thread local variables in native code
+        // require initialization/destruction. By skipping the grpc_shutdown() call,
+        // we avoid a potential crash where grpc_shutdown() has already destroyed
+        // the thread local variables, but some C core's *_destroy() methods still
+        // need to run (e.g. they may be run by finalizer thread which is out of our control)
+        // For more context, see https://github.com/grpc/grpc/issues/16294
+        private static bool IsNativeShutdownAllowed => !PlatformApis.IsXamarinIOS && !PlatformApis.IsUnityIOS;
+
         private static class ShutdownHooks
         {
             static object staticLock = new object();
diff --git a/src/csharp/Grpc.Core/Internal/PlatformApis.cs b/src/csharp/Grpc.Core/Internal/PlatformApis.cs
index c501aa89fb..a8f147545b 100644
--- a/src/csharp/Grpc.Core/Internal/PlatformApis.cs
+++ b/src/csharp/Grpc.Core/Internal/PlatformApis.cs
@@ -42,6 +42,7 @@ namespace Grpc.Core.Internal
         static readonly bool isMono;
         static readonly bool isNetCore;
         static readonly bool isUnity;
+        static readonly bool isUnityIOS;
         static readonly bool isXamarin;
         static readonly bool isXamarinIOS;
         static readonly bool isXamarinAndroid;
@@ -63,7 +64,25 @@ namespace Grpc.Core.Internal
             isNetCore = false;
 #endif
             isMono = Type.GetType("Mono.Runtime") != null;
-            isUnity = Type.GetType(UnityEngineApplicationClassName) != null;
+
+            // Unity
+            var unityApplicationClass = Type.GetType(UnityEngineApplicationClassName);
+            if (unityApplicationClass != null)
+            {
+                isUnity = true;
+                // Consult value of Application.platform via reflection
+                // https://docs.unity3d.com/ScriptReference/Application-platform.html
+                var platformProperty = unityApplicationClass.GetTypeInfo().GetProperty("platform");
+                var unityRuntimePlatform = platformProperty?.GetValue(null)?.ToString();
+                isUnityIOS = (unityRuntimePlatform == "IPhonePlayer");
+            }
+            else
+            {
+                isUnity = false;
+                isUnityIOS = false;
+            }
+
+            // Xamarin
             isXamarinIOS = Type.GetType(XamarinIOSObjectClassName) != null;
             isXamarinAndroid = Type.GetType(XamarinAndroidObjectClassName) != null;
             isXamarin = isXamarinIOS || isXamarinAndroid;
@@ -98,6 +117,14 @@ namespace Grpc.Core.Internal
         }
 
         /// <summary>
+        /// true if running on Unity iOS, false otherwise.
+        /// </summary>
+        public static bool IsUnityIOS
+        {
+            get { return isUnityIOS; }
+        }
+
+        /// <summary>
         /// true if running on a Xamarin platform (either Xamarin.Android or Xamarin.iOS),
         /// false otherwise.
         /// </summary>
diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include
index 6b0731eb40..45bd8ebd85 100755
--- a/src/csharp/Grpc.Core/Version.csproj.include
+++ b/src/csharp/Grpc.Core/Version.csproj.include
@@ -2,6 +2,6 @@
 <Project>
   <PropertyGroup>
     <GrpcCsharpVersion>1.15.0-dev</GrpcCsharpVersion>
-    <GoogleProtobufVersion>3.6.0</GoogleProtobufVersion>
+    <GoogleProtobufVersion>3.6.1</GoogleProtobufVersion>
   </PropertyGroup>
 </Project>
diff --git a/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets b/src/csharp/Grpc.Core/build/MonoAndroid10/Grpc.Core.targets
index d75e5a2f2f..250d3bd0cd 100644
--- a/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets
+++ b/src/csharp/Grpc.Core/build/MonoAndroid10/Grpc.Core.targets
@@ -1,25 +1,22 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <PropertyGroup>
-    <_GrpcCoreNugetNativePath Condition="'$(_GrpcCoreNugetNativePath)' == ''">$(MSBuildThisFileDirectory)..\..\</_GrpcCoreNugetNativePath>
-  </PropertyGroup>
 
   <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == 'MonoAndroid'">
-    <AndroidNativeLibrary Include="$(_GrpcCoreNugetNativePath)runtimes\monoandroid\arm64-v8a\libgrpc_csharp_ext.so">
+    <AndroidNativeLibrary Include="$(MSBuildThisFileDirectory)..\..\native\android\arm64-v8a\libgrpc_csharp_ext.so">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
       <Abi>arm64-v8a</Abi>
     </AndroidNativeLibrary>
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == 'MonoAndroid'">
-    <AndroidNativeLibrary Include="$(_GrpcCoreNugetNativePath)runtimes\monoandroid\armeabi-v7a\libgrpc_csharp_ext.so">
+    <AndroidNativeLibrary Include="$(MSBuildThisFileDirectory)..\..\native\android\armeabi-v7a\libgrpc_csharp_ext.so">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
       <Abi>armeabi-v7a</Abi>
     </AndroidNativeLibrary>
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == 'MonoAndroid'">
-    <AndroidNativeLibrary Include="$(_GrpcCoreNugetNativePath)runtimes\monoandroid\x86\libgrpc_csharp_ext.so">
+    <AndroidNativeLibrary Include="$(MSBuildThisFileDirectory)..\..\native\android\x86\libgrpc_csharp_ext.so">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
       <Abi>x86</Abi>
     </AndroidNativeLibrary>
diff --git a/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets b/src/csharp/Grpc.Core/build/Xamarin.iOS10/Grpc.Core.targets
index 658158f6ea..dda1cdd1e8 100644
--- a/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets
+++ b/src/csharp/Grpc.Core/build/Xamarin.iOS10/Grpc.Core.targets
@@ -2,11 +2,11 @@
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
 
   <ItemGroup>
-    <NativeReference Include="$(MSBuildThisFileDirectory)..\..\runtimes\ios\native\libgrpc_csharp_ext.a">
+    <NativeReference Include="$(MSBuildThisFileDirectory)..\..\native\ios\universal\libgrpc_csharp_ext.a">
       <Kind>Static</Kind>
       <ForceLoad>True</ForceLoad>
     </NativeReference>
-    <NativeReference Include="$(MSBuildThisFileDirectory)..\..\runtimes\ios\native\libgrpc.a">
+    <NativeReference Include="$(MSBuildThisFileDirectory)..\..\native\ios\universal\libgrpc.a">
       <Kind>Static</Kind>
       <ForceLoad>True</ForceLoad>
     </NativeReference>
diff --git a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj
index d2cc5bbc65..7493eb8051 100755
--- a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj
+++ b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj
@@ -17,8 +17,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="NUnit" Version="3.6.0" />
-    <PackageReference Include="NUnitLite" Version="3.6.0" />
+    <PackageReference Include="NUnit" Version="3.10.1" />
+    <PackageReference Include="NUnitLite" Version="3.10.1" />
     <PackageReference Include="Moq" Version="4.8.2" />
   </ItemGroup>
 
diff --git a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj
index 9da0539dcb..616e56df10 100755
--- a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj
+++ b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj
@@ -16,8 +16,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="NUnit" Version="3.6.0" />
-    <PackageReference Include="NUnitLite" Version="3.6.0" />
+    <PackageReference Include="NUnit" Version="3.10.1" />
+    <PackageReference Include="NUnitLite" Version="3.10.1" />
   </ItemGroup>
 
   <ItemGroup Condition=" '$(TargetFramework)' == 'net45' ">
diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
index e4f36d8810..ad7033b782 100755
--- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
+++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj
@@ -19,8 +19,8 @@
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="$(GoogleProtobufVersion)" />
     <PackageReference Include="CommandLineParser" Version="2.1.1-beta" />
-    <PackageReference Include="NUnit" Version="3.6.0" />
-    <PackageReference Include="NUnitLite" Version="3.6.0" />
+    <PackageReference Include="NUnit" Version="3.10.1" />
+    <PackageReference Include="NUnitLite" Version="3.10.1" />
   </ItemGroup>
 
   <ItemGroup Condition=" '$(TargetFramework)' == 'net45' ">
diff --git a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj
index d368697124..0c12f38f25 100755
--- a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj
+++ b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj
@@ -16,8 +16,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="NUnit" Version="3.6.0" />
-    <PackageReference Include="NUnitLite" Version="3.6.0" />
+    <PackageReference Include="NUnit" Version="3.10.1" />
+    <PackageReference Include="NUnitLite" Version="3.10.1" />
   </ItemGroup>
 
   <ItemGroup Condition=" '$(TargetFramework)' == 'net45' ">
diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m
index 9783b06440..084fbdeb49 100644
--- a/src/objective-c/GRPCClient/GRPCCall.m
+++ b/src/objective-c/GRPCClient/GRPCCall.m
@@ -45,6 +45,8 @@ static NSMutableDictionary *callFlags;
 static NSString *const kAuthorizationHeader = @"authorization";
 static NSString *const kBearerPrefix = @"Bearer ";
 
+const char *kCFStreamVarName = "grpc_cfstream";
+
 @interface GRPCCall ()<GRXWriteable>
 // Make them read-write.
 @property(atomic, strong) NSDictionary *responseHeaders;
@@ -206,9 +208,12 @@ static NSString *const kBearerPrefix = @"Bearer ";
   } else {
     [_responseWriteable enqueueSuccessfulCompletion];
   }
-#ifndef GRPC_CFSTREAM
-  [GRPCConnectivityMonitor unregisterObserver:self];
-#endif
+
+  // Connectivity monitor is not required for CFStream
+  char *enableCFStream = getenv(kCFStreamVarName);
+  if (enableCFStream == nil || enableCFStream[0] != '1') {
+    [GRPCConnectivityMonitor unregisterObserver:self];
+  }
 
   // If the call isn't retained anywhere else, it can be deallocated now.
   _retainSelf = nil;
@@ -220,17 +225,16 @@ static NSString *const kBearerPrefix = @"Bearer ";
 }
 
 - (void)cancel {
-  [self
-      maybeFinishWithError:[NSError
-                               errorWithDomain:kGRPCErrorDomain
-                                          code:GRPCErrorCodeCancelled
-                                      userInfo:@{NSLocalizedDescriptionKey : @"Canceled by app"}]];
-
   if (!self.isWaitingForToken) {
     [self cancelCall];
   } else {
     self.isWaitingForToken = NO;
   }
+  [self
+      maybeFinishWithError:[NSError
+                               errorWithDomain:kGRPCErrorDomain
+                                          code:GRPCErrorCodeCancelled
+                                      userInfo:@{NSLocalizedDescriptionKey : @"Canceled by app"}]];
 }
 
 - (void)maybeFinishWithError:(NSError *)errorOrNil {
@@ -292,6 +296,7 @@ static NSString *const kBearerPrefix = @"Bearer ";
         // don't want to throw, because the app shouldn't crash for a behavior
         // that's on the hands of any server to have. Instead we finish and ask
         // the server to cancel.
+        [strongSelf cancelCall];
         [strongSelf
             maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain
                                                      code:GRPCErrorCodeResourceExhausted
@@ -300,7 +305,6 @@ static NSString *const kBearerPrefix = @"Bearer ";
                                                        @"Client does not have enough memory to "
                                                        @"hold the server response."
                                                  }]];
-        [strongSelf cancelCall];
         return;
       }
       [strongWriteable enqueueValue:data
@@ -463,9 +467,11 @@ static NSString *const kBearerPrefix = @"Bearer ";
   [self sendHeaders:_requestHeaders];
   [self invokeCall];
 
-#ifndef GRPC_CFSTREAM
-  [GRPCConnectivityMonitor registerObserver:self selector:@selector(connectivityChanged:)];
-#endif
+  // Connectivity monitor is not required for CFStream
+  char *enableCFStream = getenv(kCFStreamVarName);
+  if (enableCFStream == nil || enableCFStream[0] != '1') {
+    [GRPCConnectivityMonitor registerObserver:self selector:@selector(connectivityChanged:)];
+  }
 }
 
 - (void)startWithWriteable:(id<GRXWriteable>)writeable {
@@ -530,13 +536,17 @@ static NSString *const kBearerPrefix = @"Bearer ";
 }
 
 - (void)connectivityChanged:(NSNotification *)note {
-  [self maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain
+  // Cancel underlying call upon this notification
+  __strong GRPCCall *strongSelf = self;
+  if (strongSelf) {
+    [self cancelCall];
+    [self
+        maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain
                                                  code:GRPCErrorCodeUnavailable
                                              userInfo:@{
                                                NSLocalizedDescriptionKey : @"Connectivity lost."
                                              }]];
-  // Cancel underlying call upon this notification
-  [self cancelCall];
+  }
 }
 
 @end
diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m
index bda1c3360b..f454a6dc57 100644
--- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m
+++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m
@@ -20,13 +20,8 @@
 
 #import <grpc/grpc.h>
 
-#ifdef GRPC_CFSTREAM
-const grpc_completion_queue_attributes kCompletionQueueAttr = {GRPC_CQ_CURRENT_VERSION,
-                                                               GRPC_CQ_NEXT, GRPC_CQ_NON_POLLING};
-#else
 const grpc_completion_queue_attributes kCompletionQueueAttr = {
     GRPC_CQ_CURRENT_VERSION, GRPC_CQ_NEXT, GRPC_CQ_DEFAULT_POLLING};
-#endif
 
 @implementation GRPCCompletionQueue
 
diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m
index 2e9f9f243b..862909f238 100644
--- a/src/objective-c/GRPCClient/private/GRPCHost.m
+++ b/src/objective-c/GRPCClient/private/GRPCHost.m
@@ -34,6 +34,8 @@
 
 NS_ASSUME_NONNULL_BEGIN
 
+extern const char *kCFStreamVarName;
+
 static NSMutableDictionary *kHostCache;
 
 @implementation GRPCHost {
@@ -49,9 +51,11 @@ static NSMutableDictionary *kHostCache;
   if (_channelCreds != nil) {
     grpc_channel_credentials_release(_channelCreds);
   }
-#ifndef GRPC_CFSTREAM
-  [GRPCConnectivityMonitor unregisterObserver:self];
-#endif
+  // Connectivity monitor is not required for CFStream
+  char *enableCFStream = getenv(kCFStreamVarName);
+  if (enableCFStream == nil || enableCFStream[0] != '1') {
+    [GRPCConnectivityMonitor unregisterObserver:self];
+  }
 }
 
 // Default initializer.
@@ -87,9 +91,12 @@ static NSMutableDictionary *kHostCache;
       _compressAlgorithm = GRPC_COMPRESS_NONE;
       _retryEnabled = YES;
     }
-#ifndef GRPC_CFSTREAM
-    [GRPCConnectivityMonitor registerObserver:self selector:@selector(connectivityChange:)];
-#endif
+
+    // Connectivity monitor is not required for CFStream
+    char *enableCFStream = getenv(kCFStreamVarName);
+    if (enableCFStream == nil || enableCFStream[0] != '1') {
+      [GRPCConnectivityMonitor registerObserver:self selector:@selector(connectivityChange:)];
+    }
   }
   return self;
 }
diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m
index f28e494868..7781d27ca4 100644
--- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m
+++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m
@@ -187,6 +187,7 @@
   grpc_slice _details;
   size_t _detailsCapacity;
   grpc_metadata_array _trailers;
+  const char *_errorString;
 }
 
 - (instancetype)init {
@@ -200,6 +201,7 @@
     _op.data.recv_status_on_client.status_details = &_details;
     grpc_metadata_array_init(&_trailers);
     _op.data.recv_status_on_client.trailing_metadata = &_trailers;
+    _op.data.recv_status_on_client.error_string = &_errorString;
     if (handler) {
       // Prevent reference cycle with _handler
       __weak typeof(self) weakSelf = self;
@@ -207,8 +209,9 @@
         __strong typeof(self) strongSelf = weakSelf;
         if (strongSelf) {
           char *details = grpc_slice_to_c_string(strongSelf->_details);
-          NSError *error =
-              [NSError grpc_errorFromStatusCode:strongSelf->_statusCode details:details];
+          NSError *error = [NSError grpc_errorFromStatusCode:strongSelf->_statusCode
+                                                     details:details
+                                                 errorString:strongSelf->_errorString];
           NSDictionary *trailers =
               [NSDictionary grpc_dictionaryFromMetadataArray:strongSelf->_trailers];
           handler(error, trailers);
@@ -223,6 +226,7 @@
 - (void)dealloc {
   grpc_metadata_array_destroy(&_trailers);
   grpc_slice_unref(_details);
+  gpr_free((void *)_errorString);
 }
 
 @end
diff --git a/src/objective-c/GRPCClient/private/NSError+GRPC.h b/src/objective-c/GRPCClient/private/NSError+GRPC.h
index e96b7297c2..a63e76ee4d 100644
--- a/src/objective-c/GRPCClient/private/NSError+GRPC.h
+++ b/src/objective-c/GRPCClient/private/NSError+GRPC.h
@@ -24,5 +24,7 @@
  * Returns nil if the status code is OK. Otherwise, a NSError whose code is one of |GRPCErrorCode|
  * and whose domain is |kGRPCErrorDomain|.
  */
-+ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode details:(char *)details;
++ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode
+                                 details:(char *)details
+                             errorString:(const char *)errorString;
 @end
diff --git a/src/objective-c/GRPCClient/private/NSError+GRPC.m b/src/objective-c/GRPCClient/private/NSError+GRPC.m
index c2e65e4d8a..199b2ebb6c 100644
--- a/src/objective-c/GRPCClient/private/NSError+GRPC.m
+++ b/src/objective-c/GRPCClient/private/NSError+GRPC.m
@@ -23,13 +23,19 @@
 NSString *const kGRPCErrorDomain = @"io.grpc";
 
 @implementation NSError (GRPC)
-+ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode details:(char *)details {
++ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode
+                                 details:(char *)details
+                             errorString:(const char *)errorString {
   if (statusCode == GRPC_STATUS_OK) {
     return nil;
   }
   NSString *message = [NSString stringWithCString:details encoding:NSUTF8StringEncoding];
+  NSString *debugMessage = [NSString stringWithCString:errorString encoding:NSUTF8StringEncoding];
   return [NSError errorWithDomain:kGRPCErrorDomain
                              code:statusCode
-                         userInfo:@{NSLocalizedDescriptionKey : message}];
+                         userInfo:@{
+                           NSLocalizedDescriptionKey : message,
+                           NSDebugDescriptionErrorKey : debugMessage
+                         }];
 }
 @end
diff --git a/src/objective-c/README-CFSTREAM.md b/src/objective-c/README-CFSTREAM.md
index 0b5215a7b3..0cb25ab237 100644
--- a/src/objective-c/README-CFSTREAM.md
+++ b/src/objective-c/README-CFSTREAM.md
@@ -13,17 +13,17 @@ interface that gRPC uses when it is ready for production.
 ## Usage
 If you use gRPC following the instructions in
 [README.md](https://github.com/grpc/grpc/blob/master/src/objective-c/README.md):
-- Simply replace the
-dependency on `gRPC-ProtoRPC` with `gRPC-ProtoRPC/CFStream`. The build system will take care of
-everything else and switch networking to CFStream.
+- Replace the
+dependency on `gRPC-ProtoRPC` with `gRPC-ProtoRPC/CFStream`.
+- Enable CFStream with environment variable `grpc_cfstream=1`. This can be done either in Xcode
+  console or by your code with `setenv()` before gRPC is initialized.
 
 If your project directly depends on podspecs other than `gRPC-ProtoRPC` (e.g. `gRPC` or
 `gRPC-Core`):
 
-- Make your projects depend on subspecs corresponding to CFStream in each gRPC podspec. For
-  `gRPC-Core`, you will need to make sure that the completion queue you create is of type
-  `GRPC_CQ_NON_POLLING`. This is expected to be fixed soon so that you do not have to modify the
-  completion queue type.
+- Make your projects depend on subspecs corresponding to CFStream in each gRPC podspec.
+- Enable CFStream with environment variable `grpc_cfstream=1`. This can be done either in Xcode
+  console or by your code with `setenv()` before gRPC is initialized.
 
 ## Notes
 
diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m
index 2a169800a0..a0de8ba899 100644
--- a/src/objective-c/tests/GRPCClientTests.m
+++ b/src/objective-c/tests/GRPCClientTests.m
@@ -591,4 +591,39 @@ static GRPCProtoMethod *kFullDuplexCallMethod;
   [self testTimeoutBackoffWithTimeout:0.3 Backoff:0.7];
 }
 
+- (void)testErrorDebugInformation {
+  __weak XCTestExpectation *expectation = [self expectationWithDescription:@"RPC unauthorized."];
+
+  RMTSimpleRequest *request = [RMTSimpleRequest message];
+  request.fillUsername = YES;
+  request.fillOauthScope = YES;
+  GRXWriter *requestsWriter = [GRXWriter writerWithValue:[request data]];
+
+  GRPCCall *call = [[GRPCCall alloc] initWithHost:kRemoteSSLHost
+                                             path:kUnaryCallMethod.HTTPPath
+                                   requestsWriter:requestsWriter];
+
+  call.oauth2AccessToken = @"bogusToken";
+
+  id<GRXWriteable> responsesWriteable =
+      [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) {
+        XCTFail(@"Received unexpected response: %@", value);
+      }
+          completionHandler:^(NSError *errorOrNil) {
+            XCTAssertNotNil(errorOrNil, @"Finished without error!");
+            NSDictionary *userInfo = errorOrNil.userInfo;
+            NSString *debugInformation = userInfo[NSDebugDescriptionErrorKey];
+            XCTAssertNotNil(debugInformation);
+            XCTAssertNotEqual([debugInformation length], 0);
+            NSString *challengeHeader = call.oauth2ChallengeHeader;
+            XCTAssertGreaterThan(challengeHeader.length, 0, @"No challenge in response headers %@",
+                                 call.responseHeaders);
+            [expectation fulfill];
+          }];
+
+  [call startWithWriteable:responsesWriteable];
+
+  [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil];
+}
+
 @end
diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m
index 1e1da2dd66..5750dccd89 100644
--- a/src/objective-c/tests/InteropTests.m
+++ b/src/objective-c/tests/InteropTests.m
@@ -36,6 +36,8 @@
 
 #define TEST_TIMEOUT 32
 
+extern const char *kCFStreamVarName;
+
 // Convenience constructors for the generated proto messages:
 
 @interface RMTStreamingOutputCallRequest (Constructors)
@@ -97,6 +99,9 @@ BOOL isRemoteInteropTest(NSString *host) {
   [Cronet start];
   [GRPCCall useCronetWithEngine:[Cronet getGlobalEngine]];
 #endif
+#ifdef GRPC_CFSTREAM
+  setenv(kCFStreamVarName, "1", 1);
+#endif
 }
 
 - (void)setUp {
diff --git a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj
index 8ff4633582..ea1066219d 100644
--- a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj
+++ b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj
@@ -1982,6 +1982,16 @@
 				CODE_SIGN_IDENTITY = "iPhone Developer";
 				CODE_SIGN_STYLE = Automatic;
 				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"$(inherited)",
+					"COCOAPODS=1",
+					"$(inherited)",
+					"GPB_USE_PROTOBUF_FRAMEWORK_IMPORTS=1",
+					"$(inherited)",
+					"PB_FIELD_32BIT=1",
+					"PB_NO_PACKED_STRUCTS=1",
+					"GRPC_CFSTREAM=1",
+				);
 				INFOPLIST_FILE = Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 11.2;
 				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
@@ -2100,6 +2110,16 @@
 				CODE_SIGN_IDENTITY = "iPhone Developer";
 				CODE_SIGN_STYLE = Automatic;
 				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"$(inherited)",
+					"COCOAPODS=1",
+					"$(inherited)",
+					"GPB_USE_PROTOBUF_FRAMEWORK_IMPORTS=1",
+					"$(inherited)",
+					"PB_FIELD_32BIT=1",
+					"PB_NO_PACKED_STRUCTS=1",
+					"GRPC_CFSTREAM=1",
+				);
 				INFOPLIST_FILE = Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 11.2;
 				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
@@ -2218,6 +2238,16 @@
 				CODE_SIGN_IDENTITY = "iPhone Developer";
 				CODE_SIGN_STYLE = Automatic;
 				GCC_C_LANGUAGE_STANDARD = gnu11;
+				GCC_PREPROCESSOR_DEFINITIONS = (
+					"$(inherited)",
+					"COCOAPODS=1",
+					"$(inherited)",
+					"GPB_USE_PROTOBUF_FRAMEWORK_IMPORTS=1",
+					"$(inherited)",
+					"PB_FIELD_32BIT=1",
+					"PB_NO_PACKED_STRUCTS=1",
+					"GRPC_CFSTREAM=1",
+				);
 				INFOPLIST_FILE = Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 11.2;
 				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi
index 893df8eac6..aa187e88a6 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi
@@ -309,13 +309,18 @@ cdef SegregatedCall _segregated_call(
     _ChannelState state, int flags, method, host, object deadline,
     object metadata, CallCredentials credentials, operationses_and_user_tags):
   cdef _CallState call_state = _CallState()
-  cdef grpc_completion_queue *c_completion_queue = (
-      grpc_completion_queue_create_for_next(NULL))
   cdef SegregatedCall segregated_call
+  cdef grpc_completion_queue *c_completion_queue
 
   def on_success(started_tags):
     state.segregated_call_states.add(call_state)
 
+  with state.condition:
+    if state.open:
+      c_completion_queue = (grpc_completion_queue_create_for_next(NULL))
+    else:
+      raise ValueError('Cannot invoke RPC on closed channel!')
+
   try:
     _call(
         state, call_state, c_completion_queue, on_success, flags, method, host,
@@ -443,8 +448,11 @@ cdef class Channel:
 
   def check_connectivity_state(self, bint try_to_connect):
     with self._state.condition:
-      return grpc_channel_check_connectivity_state(
-          self._state.c_channel, try_to_connect)
+      if self._state.open:
+        return grpc_channel_check_connectivity_state(
+            self._state.c_channel, try_to_connect)
+      else:
+        raise ValueError('Cannot invoke RPC on closed channel!')
 
   def watch_connectivity_state(
       self, grpc_connectivity_state last_observed_state, object deadline):
diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py
index d6efb49750..a8158311fb 100644
--- a/src/python/grpcio/grpc_core_dependencies.py
+++ b/src/python/grpcio/grpc_core_dependencies.py
@@ -254,6 +254,8 @@ CORE_SOURCE_FILES = [
     'src/core/lib/security/credentials/plugin/plugin_credentials.cc',
     'src/core/lib/security/credentials/ssl/ssl_credentials.cc',
     'src/core/lib/security/security_connector/alts_security_connector.cc',
+    'src/core/lib/security/security_connector/load_system_roots_fallback.cc',
+    'src/core/lib/security/security_connector/load_system_roots_linux.cc',
     'src/core/lib/security/security_connector/local_security_connector.cc',
     'src/core/lib/security/security_connector/security_connector.cc',
     'src/core/lib/security/transport/client_auth_filter.cc',