Merge remote-tracking branch 'upstream/master' into health_checking_service

15 files changed, 142 insertions, 50 deletions

diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc
index bb3ea400d1..23da11363b 100644
--- a/src/core/ext/filters/client_channel/client_channel.cc
+++ b/src/core/ext/filters/client_channel/client_channel.cc
@@ -129,6 +129,8 @@ typedef struct client_channel_channel_data {
   grpc_core::UniquePtr<char> info_lb_policy_name;
   /** service config in JSON form */
   grpc_core::UniquePtr<char> info_service_config_json;
+  /* backpointer to grpc_channel's channelz node */
+  grpc_core::channelz::ClientChannelNode* channelz_channel;
 } channel_data;
 
 typedef struct {
@@ -153,6 +155,23 @@ static void watch_lb_policy_locked(channel_data* chand,
                                    grpc_core::LoadBalancingPolicy* lb_policy,
                                    grpc_connectivity_state current_state);
 
+static const char* channel_connectivity_state_change_string(
+    grpc_connectivity_state state) {
+  switch (state) {
+    case GRPC_CHANNEL_IDLE:
+      return "Channel state change to IDLE";
+    case GRPC_CHANNEL_CONNECTING:
+      return "Channel state change to CONNECTING";
+    case GRPC_CHANNEL_READY:
+      return "Channel state change to READY";
+    case GRPC_CHANNEL_TRANSIENT_FAILURE:
+      return "Channel state change to TRANSIENT_FAILURE";
+    case GRPC_CHANNEL_SHUTDOWN:
+      return "Channel state change to SHUTDOWN";
+  }
+  GPR_UNREACHABLE_CODE(return "UNKNOWN");
+}
+
 static void set_channel_connectivity_state_locked(channel_data* chand,
                                                   grpc_connectivity_state state,
                                                   grpc_error* error,
@@ -177,6 +196,12 @@ static void set_channel_connectivity_state_locked(channel_data* chand,
     gpr_log(GPR_INFO, "chand=%p: setting connectivity state to %s", chand,
             grpc_connectivity_state_name(state));
   }
+  if (chand->channelz_channel != nullptr) {
+    chand->channelz_channel->AddTraceEvent(
+        grpc_core::channelz::ChannelTrace::Severity::Info,
+        grpc_slice_from_static_string(
+            channel_connectivity_state_change_string(state)));
+  }
   grpc_connectivity_state_set(&chand->state_tracker, state, error, reason);
 }
 
@@ -699,6 +724,7 @@ static grpc_error* cc_init_channel_elem(grpc_channel_element* elem,
   // Record enable_retries.
   arg = grpc_channel_args_find(args->channel_args, GRPC_ARG_ENABLE_RETRIES);
   chand->enable_retries = grpc_channel_arg_get_bool(arg, true);
+  chand->channelz_channel = nullptr;
   // Record client channel factory.
   arg = grpc_channel_args_find(args->channel_args,
                                GRPC_ARG_CLIENT_CHANNEL_FACTORY);
@@ -2194,9 +2220,9 @@ static void add_retriable_send_initial_metadata_op(
                                    .grpc_previous_rpc_attempts);
   }
   if (GPR_UNLIKELY(calld->num_attempts_completed > 0)) {
-    grpc_mdelem retry_md = grpc_mdelem_from_slices(
+    grpc_mdelem retry_md = grpc_mdelem_create(
         GRPC_MDSTR_GRPC_PREVIOUS_RPC_ATTEMPTS,
-        *retry_count_strings[calld->num_attempts_completed - 1]);
+        *retry_count_strings[calld->num_attempts_completed - 1], nullptr);
     grpc_error* error = grpc_metadata_batch_add_tail(
         &retry_state->send_initial_metadata,
         &retry_state->send_initial_metadata_storage[calld->send_initial_metadata
@@ -3208,6 +3234,12 @@ static void try_to_connect_locked(void* arg, grpc_error* error_ignored) {
   GRPC_CHANNEL_STACK_UNREF(chand->owning_stack, "try_to_connect");
 }
 
+void grpc_client_channel_set_channelz_node(
+    grpc_channel_element* elem, grpc_core::channelz::ClientChannelNode* node) {
+  channel_data* chand = static_cast<channel_data*>(elem->channel_data);
+  chand->channelz_channel = node;
+}
+
 void grpc_client_channel_populate_child_refs(
     grpc_channel_element* elem,
     grpc_core::channelz::ChildRefsList* child_subchannels,
diff --git a/src/core/ext/filters/client_channel/client_channel.h b/src/core/ext/filters/client_channel/client_channel.h
index d64faaabd2..4935fd24d8 100644
--- a/src/core/ext/filters/client_channel/client_channel.h
+++ b/src/core/ext/filters/client_channel/client_channel.h
@@ -40,6 +40,9 @@ extern grpc_core::TraceFlag grpc_client_channel_trace;
 
 extern const grpc_channel_filter grpc_client_channel_filter;
 
+void grpc_client_channel_set_channelz_node(
+    grpc_channel_element* elem, grpc_core::channelz::ClientChannelNode* node);
+
 void grpc_client_channel_populate_child_refs(
     grpc_channel_element* elem,
     grpc_core::channelz::ChildRefsList* child_subchannels,
diff --git a/src/core/ext/filters/client_channel/client_channel_channelz.cc b/src/core/ext/filters/client_channel/client_channel_channelz.cc
index b66c920b90..bad1ef668c 100644
--- a/src/core/ext/filters/client_channel/client_channel_channelz.cc
+++ b/src/core/ext/filters/client_channel/client_channel_channelz.cc
@@ -49,6 +49,7 @@ ClientChannelNode::ClientChannelNode(grpc_channel* channel,
     : ChannelNode(channel, channel_tracer_max_nodes, is_top_level_channel) {
   client_channel_ =
       grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel));
+  grpc_client_channel_set_channelz_node(client_channel_, this);
   GPR_ASSERT(client_channel_->filter == &grpc_client_channel_filter);
 }
 
diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc
index 3a1c14c6f1..a0a40eb2b3 100644
--- a/src/core/ext/filters/client_channel/subchannel.cc
+++ b/src/core/ext/filters/client_channel/subchannel.cc
@@ -428,6 +428,35 @@ intptr_t grpc_subchannel_get_child_socket_uuid(grpc_subchannel* subchannel) {
   }
 }
 
+static const char* subchannel_connectivity_state_change_string(
+    grpc_connectivity_state state) {
+  switch (state) {
+    case GRPC_CHANNEL_IDLE:
+      return "Subchannel state change to IDLE";
+    case GRPC_CHANNEL_CONNECTING:
+      return "Subchannel state change to CONNECTING";
+    case GRPC_CHANNEL_READY:
+      return "Subchannel state change to READY";
+    case GRPC_CHANNEL_TRANSIENT_FAILURE:
+      return "Subchannel state change to TRANSIENT_FAILURE";
+    case GRPC_CHANNEL_SHUTDOWN:
+      return "Subchannel state change to SHUTDOWN";
+  }
+  GPR_UNREACHABLE_CODE(return "UNKNOWN");
+}
+
+static void set_subchannel_connectivity_state_locked(
+    grpc_subchannel* c, grpc_connectivity_state state, grpc_error* error,
+    const char* reason) {
+  if (c->channelz_subchannel != nullptr) {
+    c->channelz_subchannel->AddTraceEvent(
+        grpc_core::channelz::ChannelTrace::Severity::Info,
+        grpc_slice_from_static_string(
+            subchannel_connectivity_state_change_string(state)));
+  }
+  grpc_connectivity_state_set(&c->state_tracker, state, error, reason);
+}
+
 static void continue_connect_locked(grpc_subchannel* c) {
   grpc_connect_in_args args;
   args.interested_parties = c->pollset_set;
@@ -436,8 +465,8 @@ static void continue_connect_locked(grpc_subchannel* c) {
   c->next_attempt_deadline = c->backoff->NextAttemptTime();
   args.deadline = std::max(c->next_attempt_deadline, min_deadline);
   args.channel_args = c->args;
-  grpc_connectivity_state_set(&c->state_tracker, GRPC_CHANNEL_CONNECTING,
-                              GRPC_ERROR_NONE, "connecting");
+  set_subchannel_connectivity_state_locked(c, GRPC_CHANNEL_CONNECTING,
+                                           GRPC_ERROR_NONE, "connecting");
   grpc_connector_connect(c->connector, &args, &c->connecting_result,
                          &c->on_connected);
 }
@@ -587,9 +616,9 @@ static void on_connected_subchannel_connectivity_changed(void* p,
                       connected_subchannel_watcher->connectivity_state));
         }
         c->connected_subchannel.reset();
-        grpc_connectivity_state_set(&c->state_tracker,
-                                    GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                    GRPC_ERROR_REF(error), "reflect_child");
+        set_subchannel_connectivity_state_locked(
+            c, GRPC_CHANNEL_TRANSIENT_FAILURE, GRPC_ERROR_REF(error),
+            "reflect_child");
         c->backoff_begun = false;
         c->backoff->Reset();
         maybe_start_connecting_locked(c);
@@ -600,8 +629,8 @@ static void on_connected_subchannel_connectivity_changed(void* p,
       break;
     }
     default: {
-      grpc_connectivity_state_set(
-          &c->state_tracker, connected_subchannel_watcher->connectivity_state,
+      set_subchannel_connectivity_state_locked(
+          c, connected_subchannel_watcher->connectivity_state,
           GRPC_ERROR_REF(error), "reflect_child");
       GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher");
       c->connected_subchannel->NotifyOnStateChange(
@@ -672,8 +701,8 @@ static bool publish_transport_locked(grpc_subchannel* c) {
       &connected_subchannel_watcher->closure);
 
   /* signal completion */
-  grpc_connectivity_state_set(&c->state_tracker, GRPC_CHANNEL_READY,
-                              GRPC_ERROR_NONE, "connected");
+  set_subchannel_connectivity_state_locked(c, GRPC_CHANNEL_READY,
+                                           GRPC_ERROR_NONE, "connected");
   return true;
 }
 
@@ -690,8 +719,8 @@ static void on_subchannel_connected(void* arg, grpc_error* error) {
   } else if (c->disconnected) {
     GRPC_SUBCHANNEL_WEAK_UNREF(c, "connecting");
   } else {
-    grpc_connectivity_state_set(
-        &c->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE,
+    set_subchannel_connectivity_state_locked(
+        c, GRPC_CHANNEL_TRANSIENT_FAILURE,
         grpc_error_set_int(GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
                                "Connect Failed", &error, 1),
                            GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE),
@@ -760,9 +789,12 @@ static void get_call_status(grpc_subchannel_call* call,
     grpc_error_get_status(error, call->deadline, status, nullptr, nullptr,
                           nullptr);
   } else {
-    GPR_ASSERT(md_batch->idx.named.grpc_status != nullptr);
-    *status =
-        grpc_get_status_code_from_metadata(md_batch->idx.named.grpc_status->md);
+    if (md_batch->idx.named.grpc_status != nullptr) {
+      *status = grpc_get_status_code_from_metadata(
+          md_batch->idx.named.grpc_status->md);
+    } else {
+      *status = GRPC_STATUS_UNKNOWN;
+    }
   }
   GRPC_ERROR_UNREF(error);
 }
diff --git a/src/core/ext/filters/http/client_authority_filter.cc b/src/core/ext/filters/http/client_authority_filter.cc
index 1ca20ebb26..6383f12594 100644
--- a/src/core/ext/filters/http/client_authority_filter.cc
+++ b/src/core/ext/filters/http/client_authority_filter.cc
@@ -59,9 +59,8 @@ void authority_start_transport_stream_op_batch(
       initial_metadata->idx.named.authority == nullptr) {
     grpc_error* error = grpc_metadata_batch_add_head(
         initial_metadata, &calld->authority_storage,
-        grpc_mdelem_from_slices(
-            GRPC_MDSTR_AUTHORITY,
-            grpc_slice_ref_internal(chand->default_authority)));
+        grpc_mdelem_create(GRPC_MDSTR_AUTHORITY, chand->default_authority,
+                           nullptr));
     if (error != GRPC_ERROR_NONE) {
       grpc_transport_stream_op_batch_finish_with_failure(batch, error,
                                                          calld->call_combiner);
diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index 2d4b4da4c6..8a481bb7d5 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -2128,8 +2128,7 @@ void grpc_chttp2_fake_status(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
           "add_status_message",
           grpc_chttp2_incoming_metadata_buffer_replace_or_add(
               &s->metadata_buffer[1],
-              grpc_mdelem_from_slices(GRPC_MDSTR_GRPC_MESSAGE,
-                                      grpc_slice_ref_internal(slice))));
+              grpc_mdelem_create(GRPC_MDSTR_GRPC_MESSAGE, slice, nullptr)));
     }
     s->published_metadata[1] = GRPC_METADATA_SYNTHESIZED_FROM_FAKE;
     grpc_chttp2_maybe_complete_recv_trailing_metadata(t, s);
diff --git a/src/core/lib/channel/channelz.h b/src/core/lib/channel/channelz.h
index fddef793fb..88551befc8 100644
--- a/src/core/lib/channel/channelz.h
+++ b/src/core/lib/channel/channelz.h
@@ -42,13 +42,13 @@
 
 /** This is the default value for whether or not to enable channelz. If
  * GRPC_ARG_ENABLE_CHANNELZ is set, it will override this default value. */
-#define GRPC_ENABLE_CHANNELZ_DEFAULT false
+#define GRPC_ENABLE_CHANNELZ_DEFAULT true
 
 /** This is the default value for the maximum amount of memory used by trace
  * events per channel trace node. If
  * GRPC_ARG_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE is set, it will override
  * this default value. */
-#define GRPC_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE_DEFAULT 0
+#define GRPC_MAX_CHANNEL_TRACE_EVENT_MEMORY_PER_NODE_DEFAULT 1024 * 4
 
 namespace grpc_core {
 
diff --git a/src/core/lib/iomgr/wakeup_fd_eventfd.cc b/src/core/lib/iomgr/wakeup_fd_eventfd.cc
index dcf7dab71f..d68c9ada1f 100644
--- a/src/core/lib/iomgr/wakeup_fd_eventfd.cc
+++ b/src/core/lib/iomgr/wakeup_fd_eventfd.cc
@@ -32,12 +32,11 @@
 #include "src/core/lib/profiling/timers.h"
 
 static grpc_error* eventfd_create(grpc_wakeup_fd* fd_info) {
-  int efd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
-  if (efd < 0) {
+  fd_info->read_fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
+  fd_info->write_fd = -1;
+  if (fd_info->read_fd < 0) {
     return GRPC_OS_ERROR(errno, "eventfd");
   }
-  fd_info->read_fd = efd;
-  fd_info->write_fd = -1;
   return GRPC_ERROR_NONE;
 }
 
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.cc b/src/core/lib/security/credentials/plugin/plugin_credentials.cc
index 73946ce039..4015124298 100644
--- a/src/core/lib/security/credentials/plugin/plugin_credentials.cc
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.cc
@@ -102,8 +102,7 @@ static grpc_error* process_plugin_result(
     } else {
       for (size_t i = 0; i < num_md; ++i) {
         grpc_mdelem mdelem =
-            grpc_mdelem_from_slices(grpc_slice_ref_internal(md[i].key),
-                                    grpc_slice_ref_internal(md[i].value));
+            grpc_mdelem_create(md[i].key, md[i].value, nullptr);
         grpc_credentials_mdelem_array_add(r->md_array, mdelem);
         GRPC_MDELEM_UNREF(mdelem);
       }
diff --git a/src/core/lib/surface/channel.cc b/src/core/lib/surface/channel.cc
index 4294434504..d7095c24d4 100644
--- a/src/core/lib/surface/channel.cc
+++ b/src/core/lib/surface/channel.cc
@@ -336,9 +336,8 @@ grpc_call* grpc_channel_create_call(grpc_channel* channel,
   grpc_core::ExecCtx exec_ctx;
   grpc_call* call = grpc_channel_create_call_internal(
       channel, parent_call, propagation_mask, cq, nullptr,
-      grpc_mdelem_from_slices(GRPC_MDSTR_PATH, grpc_slice_ref_internal(method)),
-      host != nullptr ? grpc_mdelem_from_slices(GRPC_MDSTR_AUTHORITY,
-                                                grpc_slice_ref_internal(*host))
+      grpc_mdelem_create(GRPC_MDSTR_PATH, method, nullptr),
+      host != nullptr ? grpc_mdelem_create(GRPC_MDSTR_AUTHORITY, *host, nullptr)
                       : GRPC_MDNULL,
       grpc_timespec_to_millis_round_up(deadline));
 
@@ -347,14 +346,13 @@ grpc_call* grpc_channel_create_call(grpc_channel* channel,
 
 grpc_call* grpc_channel_create_pollset_set_call(
     grpc_channel* channel, grpc_call* parent_call, uint32_t propagation_mask,
-    grpc_pollset_set* pollset_set, grpc_slice method, const grpc_slice* host,
-    grpc_millis deadline, void* reserved) {
+    grpc_pollset_set* pollset_set, const grpc_slice& method,
+    const grpc_slice* host, grpc_millis deadline, void* reserved) {
   GPR_ASSERT(!reserved);
   return grpc_channel_create_call_internal(
       channel, parent_call, propagation_mask, nullptr, pollset_set,
-      grpc_mdelem_from_slices(GRPC_MDSTR_PATH, grpc_slice_ref_internal(method)),
-      host != nullptr ? grpc_mdelem_from_slices(GRPC_MDSTR_AUTHORITY,
-                                                grpc_slice_ref_internal(*host))
+      grpc_mdelem_create(GRPC_MDSTR_PATH, method, nullptr),
+      host != nullptr ? grpc_mdelem_create(GRPC_MDSTR_AUTHORITY, *host, nullptr)
                       : GRPC_MDNULL,
       deadline);
 }
diff --git a/src/core/lib/surface/channel.h b/src/core/lib/surface/channel.h
index e5ff2c3596..4ac76b8a29 100644
--- a/src/core/lib/surface/channel.h
+++ b/src/core/lib/surface/channel.h
@@ -45,8 +45,8 @@ grpc_channel* grpc_channel_create_with_builder(
     value of \a propagation_mask (see propagation_bits.h for possible values) */
 grpc_call* grpc_channel_create_pollset_set_call(
     grpc_channel* channel, grpc_call* parent_call, uint32_t propagation_mask,
-    grpc_pollset_set* pollset_set, grpc_slice method, const grpc_slice* host,
-    grpc_millis deadline, void* reserved);
+    grpc_pollset_set* pollset_set, const grpc_slice& method,
+    const grpc_slice* host, grpc_millis deadline, void* reserved);
 
 /** Get a (borrowed) pointer to this channels underlying channel stack */
 grpc_channel_stack* grpc_channel_get_channel_stack(grpc_channel* channel);
diff --git a/src/core/lib/transport/metadata.cc b/src/core/lib/transport/metadata.cc
index d164502280..60af22393e 100644
--- a/src/core/lib/transport/metadata.cc
+++ b/src/core/lib/transport/metadata.cc
@@ -237,7 +237,7 @@ static void rehash_mdtab(mdtab_shard* shard) {
 }
 
 grpc_mdelem grpc_mdelem_create(
-    grpc_slice key, grpc_slice value,
+    const grpc_slice& key, const grpc_slice& value,
     grpc_mdelem_data* compatible_external_backing_store) {
   if (!grpc_slice_is_interned(key) || !grpc_slice_is_interned(value)) {
     if (compatible_external_backing_store != nullptr) {
@@ -324,7 +324,8 @@ grpc_mdelem grpc_mdelem_create(
   return GRPC_MAKE_MDELEM(md, GRPC_MDELEM_STORAGE_INTERNED);
 }
 
-grpc_mdelem grpc_mdelem_from_slices(grpc_slice key, grpc_slice value) {
+grpc_mdelem grpc_mdelem_from_slices(const grpc_slice& key,
+                                    const grpc_slice& value) {
   grpc_mdelem out = grpc_mdelem_create(key, value, nullptr);
   grpc_slice_unref_internal(key);
   grpc_slice_unref_internal(value);
diff --git a/src/core/lib/transport/metadata.h b/src/core/lib/transport/metadata.h
index 338082276c..989c7544c1 100644
--- a/src/core/lib/transport/metadata.h
+++ b/src/core/lib/transport/metadata.h
@@ -109,7 +109,8 @@ struct grpc_mdelem {
                               (uintptr_t)GRPC_MDELEM_STORAGE_INTERNED_BIT))
 
 /* Unrefs the slices. */
-grpc_mdelem grpc_mdelem_from_slices(grpc_slice key, grpc_slice value);
+grpc_mdelem grpc_mdelem_from_slices(const grpc_slice& key,
+                                    const grpc_slice& value);
 
 /* Cheaply convert a grpc_metadata to a grpc_mdelem; may use the grpc_metadata
    object as backing storage (so lifetimes should align) */
@@ -120,7 +121,7 @@ grpc_mdelem grpc_mdelem_from_grpc_metadata(grpc_metadata* metadata);
    compatible_external_backing_store if it is non-NULL (in which case it's the
    users responsibility to ensure that it outlives usage) */
 grpc_mdelem grpc_mdelem_create(
-    grpc_slice key, grpc_slice value,
+    const grpc_slice& key, const grpc_slice& value,
     grpc_mdelem_data* compatible_external_backing_store);
 
 bool grpc_mdelem_eq(grpc_mdelem a, grpc_mdelem b);
diff --git a/src/csharp/Grpc.Core/Internal/UnmanagedLibrary.cs b/src/csharp/Grpc.Core/Internal/UnmanagedLibrary.cs
index 7185d68efe..1786fc2e3f 100644
--- a/src/csharp/Grpc.Core/Internal/UnmanagedLibrary.cs
+++ b/src/csharp/Grpc.Core/Internal/UnmanagedLibrary.cs
@@ -51,11 +51,12 @@ namespace Grpc.Core.Internal
 
             Logger.Debug("Attempting to load native library \"{0}\"", this.libraryPath);
 
-            this.handle = PlatformSpecificLoadLibrary(this.libraryPath);
+            this.handle = PlatformSpecificLoadLibrary(this.libraryPath, out string loadLibraryErrorDetail);
 
             if (this.handle == IntPtr.Zero)
             {
-                throw new IOException(string.Format("Error loading native library \"{0}\"", this.libraryPath));
+                throw new IOException(string.Format("Error loading native library \"{0}\". {1}",
+                                                    this.libraryPath, loadLibraryErrorDetail));
             }
         }
 
@@ -129,31 +130,44 @@ namespace Grpc.Core.Internal
         /// <summary>
         /// Loads library in a platform specific way.
         /// </summary>
-        private static IntPtr PlatformSpecificLoadLibrary(string libraryPath)
+        private static IntPtr PlatformSpecificLoadLibrary(string libraryPath, out string errorMsg)
         {
             if (PlatformApis.IsWindows)
             {
+                // TODO(jtattermusch): populate the error on Windows
+                errorMsg = null;
                 return Windows.LoadLibrary(libraryPath);
             }
             if (PlatformApis.IsLinux)
             {
                 if (PlatformApis.IsMono)
                 {
-                    return Mono.dlopen(libraryPath, RTLD_GLOBAL + RTLD_LAZY);
+                    return LoadLibraryPosix(Mono.dlopen, Mono.dlerror, libraryPath, out errorMsg);
                 }
                 if (PlatformApis.IsNetCore)
                 {
-                    return CoreCLR.dlopen(libraryPath, RTLD_GLOBAL + RTLD_LAZY);
+                    return LoadLibraryPosix(CoreCLR.dlopen, CoreCLR.dlerror, libraryPath, out errorMsg);
                 }
-                return Linux.dlopen(libraryPath, RTLD_GLOBAL + RTLD_LAZY);
+                return LoadLibraryPosix(Linux.dlopen, Linux.dlerror, libraryPath, out errorMsg);
             }
             if (PlatformApis.IsMacOSX)
             {
-                return MacOSX.dlopen(libraryPath, RTLD_GLOBAL + RTLD_LAZY);
+                return LoadLibraryPosix(MacOSX.dlopen, MacOSX.dlerror, libraryPath, out errorMsg);
             }
             throw new InvalidOperationException("Unsupported platform.");
         }
 
+        private static IntPtr LoadLibraryPosix(Func<string, int, IntPtr> dlopenFunc, Func<IntPtr> dlerrorFunc, string libraryPath, out string errorMsg)
+        {
+            errorMsg = null;
+            IntPtr ret = dlopenFunc(libraryPath, RTLD_GLOBAL + RTLD_LAZY);
+            if (ret == IntPtr.Zero)
+            {
+                errorMsg = Marshal.PtrToStringAnsi(dlerrorFunc());
+            }
+            return ret;
+        }
+
         private static string FirstValidLibraryPath(string[] libraryPathAlternatives)
         {
             GrpcPreconditions.CheckArgument(libraryPathAlternatives.Length > 0, "libraryPathAlternatives cannot be empty.");
@@ -184,6 +198,9 @@ namespace Grpc.Core.Internal
             internal static extern IntPtr dlopen(string filename, int flags);
 
             [DllImport("libdl.so")]
+            internal static extern IntPtr dlerror();
+
+            [DllImport("libdl.so")]
             internal static extern IntPtr dlsym(IntPtr handle, string symbol);
         }
 
@@ -193,6 +210,9 @@ namespace Grpc.Core.Internal
             internal static extern IntPtr dlopen(string filename, int flags);
 
             [DllImport("libSystem.dylib")]
+            internal static extern IntPtr dlerror();
+
+            [DllImport("libSystem.dylib")]
             internal static extern IntPtr dlsym(IntPtr handle, string symbol);
         }
 
@@ -209,6 +229,9 @@ namespace Grpc.Core.Internal
             internal static extern IntPtr dlopen(string filename, int flags);
 
             [DllImport("__Internal")]
+            internal static extern IntPtr dlerror();
+
+            [DllImport("__Internal")]
             internal static extern IntPtr dlsym(IntPtr handle, string symbol);
         }
 
@@ -223,6 +246,9 @@ namespace Grpc.Core.Internal
             internal static extern IntPtr dlopen(string filename, int flags);
 
             [DllImport("libcoreclr.so")]
+            internal static extern IntPtr dlerror();
+
+            [DllImport("libcoreclr.so")]
             internal static extern IntPtr dlsym(IntPtr handle, string symbol);
         }
     }
diff --git a/src/python/grpcio_tests/commands.py b/src/python/grpcio_tests/commands.py
index 0dfbf3180b..6931d93ef0 100644
--- a/src/python/grpcio_tests/commands.py
+++ b/src/python/grpcio_tests/commands.py
@@ -116,6 +116,8 @@ class TestGevent(setuptools.Command):
         # eventually succeed, but need to dig into performance issues.
         'unit._cython._no_messages_server_completion_queue_per_call_test.Test.test_rpcs',
         'unit._cython._no_messages_single_server_completion_queue_test.Test.test_rpcs',
+        # TODO(https://github.com/grpc/grpc/issues/16890) enable this test
+        'unit._cython._channel_test.ChannelTest.test_multiple_channels_lonely_connectivity',
         # I have no idea why this doesn't work in gevent, but it shouldn't even be
         # using the c-core
         'testing._client_test.ClientTest.test_infinite_request_stream_real_time',