Merge github.com:grpc/grpc into can-we-be-single-threaded

28 files changed, 160 insertions, 100 deletions

diff --git a/src/core/census/grpc_filter.c b/src/core/census/grpc_filter.c
index 1830af3feb..3545307b66 100644
--- a/src/core/census/grpc_filter.c
+++ b/src/core/census/grpc_filter.c
@@ -168,7 +168,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
                               int is_first, int is_last) {
   channel_data *chand = elem->channel_data;
   GPR_ASSERT(chand != NULL);
-  chand->path_str = grpc_mdstr_from_string(mdctx, ":path", 0);
+  chand->path_str = grpc_mdstr_from_string(mdctx, ":path");
 }
 
 static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,
diff --git a/src/core/channel/compress_filter.c b/src/core/channel/compress_filter.c
index f8dbe8c817..2aa826fcd8 100644
--- a/src/core/channel/compress_filter.c
+++ b/src/core/channel/compress_filter.c
@@ -331,13 +331,13 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
       channeld->default_compression_algorithm;
 
   channeld->mdstr_request_compression_algorithm_key =
-      grpc_mdstr_from_string(mdctx, GRPC_COMPRESS_REQUEST_ALGORITHM_KEY, 0);
+      grpc_mdstr_from_string(mdctx, GRPC_COMPRESS_REQUEST_ALGORITHM_KEY);
 
   channeld->mdstr_outgoing_compression_algorithm_key =
-      grpc_mdstr_from_string(mdctx, "grpc-encoding", 0);
+      grpc_mdstr_from_string(mdctx, "grpc-encoding");
 
   channeld->mdstr_compression_capabilities_key =
-      grpc_mdstr_from_string(mdctx, "grpc-accept-encoding", 0);
+      grpc_mdstr_from_string(mdctx, "grpc-accept-encoding");
 
   for (algo_idx = 0; algo_idx < GRPC_COMPRESS_ALGORITHMS_COUNT; ++algo_idx) {
     char *algorithm_name;
@@ -351,7 +351,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
         grpc_mdelem_from_metadata_strings(
             mdctx,
             GRPC_MDSTR_REF(channeld->mdstr_outgoing_compression_algorithm_key),
-            grpc_mdstr_from_string(mdctx, algorithm_name, 0));
+            grpc_mdstr_from_string(mdctx, algorithm_name));
     if (algo_idx > 0) {
       supported_algorithms_names[supported_algorithms_idx++] = algorithm_name;
     }
@@ -365,7 +365,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
 
   channeld->mdelem_accept_encoding = grpc_mdelem_from_metadata_strings(
       mdctx, GRPC_MDSTR_REF(channeld->mdstr_compression_capabilities_key),
-      grpc_mdstr_from_string(mdctx, accept_encoding_str, 0));
+      grpc_mdstr_from_string(mdctx, accept_encoding_str));
   gpr_free(accept_encoding_str);
 
   GPR_ASSERT(!is_last);
diff --git a/src/core/channel/http_client_filter.c b/src/core/channel/http_client_filter.c
index da33c956f7..d67dc37ad2 100644
--- a/src/core/channel/http_client_filter.c
+++ b/src/core/channel/http_client_filter.c
@@ -239,7 +239,7 @@ static grpc_mdstr *user_agent_from_args(grpc_mdctx *mdctx,
 
   tmp = gpr_strvec_flatten(&v, NULL);
   gpr_strvec_destroy(&v);
-  result = grpc_mdstr_from_string(mdctx, tmp, 0);
+  result = grpc_mdstr_from_string(mdctx, tmp);
   gpr_free(tmp);
 
   return result;
@@ -267,7 +267,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
       grpc_mdelem_from_strings(mdctx, "content-type", "application/grpc");
   channeld->status = grpc_mdelem_from_strings(mdctx, ":status", "200");
   channeld->user_agent = grpc_mdelem_from_metadata_strings(
-      mdctx, grpc_mdstr_from_string(mdctx, "user-agent", 0),
+      mdctx, grpc_mdstr_from_string(mdctx, "user-agent"),
       user_agent_from_args(mdctx, channel_args));
 }
 
diff --git a/src/core/channel/http_server_filter.c b/src/core/channel/http_server_filter.c
index 549c1da580..5e6d684a52 100644
--- a/src/core/channel/http_server_filter.c
+++ b/src/core/channel/http_server_filter.c
@@ -273,9 +273,9 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
   channeld->http_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "http");
   channeld->https_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "https");
   channeld->grpc_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "grpc");
-  channeld->path_key = grpc_mdstr_from_string(mdctx, ":path", 0);
-  channeld->authority_key = grpc_mdstr_from_string(mdctx, ":authority", 0);
-  channeld->host_key = grpc_mdstr_from_string(mdctx, "host", 0);
+  channeld->path_key = grpc_mdstr_from_string(mdctx, ":path");
+  channeld->authority_key = grpc_mdstr_from_string(mdctx, ":authority");
+  channeld->host_key = grpc_mdstr_from_string(mdctx, "host");
   channeld->content_type =
       grpc_mdelem_from_strings(mdctx, "content-type", "application/grpc");
 
diff --git a/src/core/client_config/subchannel.h b/src/core/client_config/subchannel.h
index c9e5861d9c..2af3ce9e56 100644
--- a/src/core/client_config/subchannel.h
+++ b/src/core/client_config/subchannel.h
@@ -64,13 +64,13 @@ typedef struct grpc_subchannel_args grpc_subchannel_args;
 #define GRPC_SUBCHANNEL_REF_EXTRA_ARGS
 #endif
 
-void grpc_subchannel_ref(
-    grpc_subchannel *channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+void grpc_subchannel_ref(grpc_subchannel *channel
+                             GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_unref(grpc_exec_ctx *exec_ctx,
                            grpc_subchannel *channel
                                GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
-void grpc_subchannel_call_ref(
-    grpc_subchannel_call *call GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
+void grpc_subchannel_call_ref(grpc_subchannel_call *call
+                                  GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_subchannel_call_unref(grpc_exec_ctx *exec_ctx,
                                 grpc_subchannel_call *call
                                     GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c
index 5626b08a47..a4293eb4a4 100644
--- a/src/core/iomgr/pollset_multipoller_with_epoll.c
+++ b/src/core/iomgr/pollset_multipoller_with_epoll.c
@@ -42,6 +42,7 @@
 #include <unistd.h>
 
 #include "src/core/iomgr/fd_posix.h"
+#include "src/core/support/block_annotate.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
@@ -179,7 +180,9 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock(
   pfds[1].events = POLLIN;
   pfds[1].revents = 0;
 
+  GRPC_SCHEDULING_START_BLOCKING_REGION;
   poll_rv = grpc_poll_function(pfds, 2, timeout_ms);
+  GRPC_SCHEDULING_END_BLOCKING_REGION;
 
   if (poll_rv < 0) {
     if (errno != EINTR) {
@@ -193,6 +196,7 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock(
     }
     if (pfds[1].revents) {
       do {
+	/* The following epoll_wait never blocks; it has a timeout of 0 */
         ep_rv = epoll_wait(h->epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, 0);
         if (ep_rv < 0) {
           if (errno != EINTR) {
diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c
index 2a18cedb33..44031b8ef6 100644
--- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c
+++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c
@@ -44,6 +44,7 @@
 
 #include "src/core/iomgr/fd_posix.h"
 #include "src/core/iomgr/iomgr_internal.h"
+#include "src/core/support/block_annotate.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
@@ -147,7 +148,9 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock(
                                                POLLOUT, &watchers[i]);
   }
 
+  GRPC_SCHEDULING_START_BLOCKING_REGION;
   r = grpc_poll_function(pfds, pfd_count, timeout);
+  GRPC_SCHEDULING_END_BLOCKING_REGION;
 
   for (i = 1; i < pfd_count; i++) {
     grpc_fd_end_poll(exec_ctx, &watchers[i], pfds[i].revents & POLLIN,
diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c
index 43ad22c16d..e80963e0ea 100644
--- a/src/core/iomgr/pollset_posix.c
+++ b/src/core/iomgr/pollset_posix.c
@@ -47,6 +47,7 @@
 #include "src/core/iomgr/iomgr_internal.h"
 #include "src/core/iomgr/socket_utils_posix.h"
 #include "src/core/profiling/timers.h"
+#include "src/core/support/block_annotate.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/thd.h>
@@ -118,11 +119,13 @@ void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) {
 
 void grpc_pollset_global_init(void) {
   gpr_tls_init(&g_current_thread_poller);
+  gpr_tls_init(&g_current_thread_worker);
   grpc_wakeup_fd_global_init();
 }
 
 void grpc_pollset_global_shutdown(void) {
   gpr_tls_destroy(&g_current_thread_poller);
+  gpr_tls_destroy(&g_current_thread_worker);
   grpc_wakeup_fd_global_destroy();
 }
 
@@ -468,7 +471,9 @@ static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx,
 
   /* poll fd count (argument 2) is shortened by one if we have no events
      to poll on - such that it only includes the kicker */
+  GRPC_SCHEDULING_START_BLOCKING_REGION;
   r = grpc_poll_function(pfd, nfds, timeout);
+  GRPC_SCHEDULING_END_BLOCKING_REGION;
   GRPC_TIMER_MARK(GRPC_PTAG_POLL_FINISHED, r);
 
   if (fd) {
diff --git a/src/core/iomgr/resolve_address_posix.c b/src/core/iomgr/resolve_address_posix.c
index 99bd566e10..ed0a93fcc9 100644
--- a/src/core/iomgr/resolve_address_posix.c
+++ b/src/core/iomgr/resolve_address_posix.c
@@ -43,6 +43,7 @@
 
 #include "src/core/iomgr/iomgr_internal.h"
 #include "src/core/iomgr/sockaddr_utils.h"
+#include "src/core/support/block_annotate.h"
 #include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/host_port.h>
@@ -103,13 +104,18 @@ grpc_resolved_addresses *grpc_blocking_resolve_address(
   hints.ai_socktype = SOCK_STREAM; /* stream socket */
   hints.ai_flags = AI_PASSIVE;     /* for wildcard IP address */
 
+  GRPC_SCHEDULING_START_BLOCKING_REGION;
   s = getaddrinfo(host, port, &hints, &result);
+  GRPC_SCHEDULING_END_BLOCKING_REGION;
+
   if (s != 0) {
     /* Retry if well-known service name is recognized */
     char *svc[][2] = {{"http", "80"}, {"https", "443"}};
     for (i = 0; i < GPR_ARRAY_SIZE(svc); i++) {
       if (strcmp(port, svc[i][0]) == 0) {
+        GRPC_SCHEDULING_START_BLOCKING_REGION;
         s = getaddrinfo(host, svc[i][1], &hints, &result);
+        GRPC_SCHEDULING_END_BLOCKING_REGION;
         break;
       }
     }
diff --git a/src/core/iomgr/resolve_address_windows.c b/src/core/iomgr/resolve_address_windows.c
index fcd80b3912..82a5602996 100644
--- a/src/core/iomgr/resolve_address_windows.c
+++ b/src/core/iomgr/resolve_address_windows.c
@@ -42,6 +42,7 @@
 
 #include "src/core/iomgr/iomgr_internal.h"
 #include "src/core/iomgr/sockaddr_utils.h"
+#include "src/core/support/block_annotate.h"
 #include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/host_port.h>
@@ -88,7 +89,9 @@ grpc_resolved_addresses *grpc_blocking_resolve_address(
   hints.ai_socktype = SOCK_STREAM; /* stream socket */
   hints.ai_flags = AI_PASSIVE;     /* for wildcard IP address */
 
+  GRPC_SCHEDULING_START_BLOCKING_REGION;
   s = getaddrinfo(host, port, &hints, &result);
+  GRPC_SCHEDULING_END_BLOCKING_REGION;
   if (s != 0) {
     gpr_log(GPR_ERROR, "getaddrinfo: %s", gai_strerror(s));
     goto done;
diff --git a/src/core/iomgr/tcp_posix.c b/src/core/iomgr/tcp_posix.c
index c98f0125f8..54ebad7dbc 100644
--- a/src/core/iomgr/tcp_posix.c
+++ b/src/core/iomgr/tcp_posix.c
@@ -78,6 +78,9 @@ typedef struct {
   size_t slice_size;
   gpr_refcount refcount;
 
+  /* garbage after the last read */
+  gpr_slice_buffer last_read_buffer;
+
   gpr_slice_buffer *incoming_buffer;
   gpr_slice_buffer *outgoing_buffer;
   /** slice within outgoing_buffer to write next */
@@ -106,6 +109,7 @@ static void tcp_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) {
 
 static void tcp_free(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
   grpc_fd_orphan(exec_ctx, tcp->em_fd, NULL, "tcp_unref_orphan");
+  gpr_slice_buffer_destroy(&tcp->last_read_buffer);
   gpr_free(tcp->peer_string);
   gpr_free(tcp);
 }
@@ -226,7 +230,8 @@ static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
     if ((size_t)read_bytes < tcp->incoming_buffer->length) {
       gpr_slice_buffer_trim_end(
           tcp->incoming_buffer,
-          tcp->incoming_buffer->length - (size_t)read_bytes);
+          tcp->incoming_buffer->length - (size_t)read_bytes,
+          &tcp->last_read_buffer);
     } else if (tcp->iov_size < MAX_READ_IOVEC) {
       ++tcp->iov_size;
     }
@@ -259,6 +264,7 @@ static void tcp_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   tcp->read_cb = cb;
   tcp->incoming_buffer = incoming_buffer;
   gpr_slice_buffer_reset_and_unref(incoming_buffer);
+  gpr_slice_buffer_swap(incoming_buffer, &tcp->last_read_buffer);
   TCP_REF(tcp, "read");
   if (tcp->finished_edge) {
     tcp->finished_edge = 0;
@@ -457,6 +463,7 @@ grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size,
   tcp->read_closure.cb_arg = tcp;
   tcp->write_closure.cb = tcp_handle_write;
   tcp->write_closure.cb_arg = tcp;
+  gpr_slice_buffer_init(&tcp->last_read_buffer);
 
   return &tcp->base;
 }
diff --git a/src/core/iomgr/udp_server.c b/src/core/iomgr/udp_server.c
index ae7c889d0c..9baaf1edc7 100644
--- a/src/core/iomgr/udp_server.c
+++ b/src/core/iomgr/udp_server.c
@@ -117,6 +117,8 @@ struct grpc_udp_server {
   grpc_pollset **pollsets;
   /* number of pollsets in the pollsets array */
   size_t pollset_count;
+  /* The parent grpc server */
+  grpc_server* grpc_server;
 };
 
 grpc_udp_server *grpc_udp_server_create(void) {
@@ -276,7 +278,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, int success) {
 
   /* Tell the registered callback that data is available to read. */
   GPR_ASSERT(sp->read_cb);
-  sp->read_cb(sp->fd);
+  sp->read_cb(sp->fd, sp->server->grpc_server);
 
   /* Re-arm the notification event so we get another chance to read. */
   grpc_fd_notify_on_read(exec_ctx, sp->emfd, &sp->read_closure);
@@ -402,11 +404,13 @@ int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned index) {
 }
 
 void grpc_udp_server_start(grpc_exec_ctx *exec_ctx, grpc_udp_server *s,
-                           grpc_pollset **pollsets, size_t pollset_count) {
+                           grpc_pollset **pollsets, size_t pollset_count,
+                           grpc_server *server) {
   size_t i, j;
   gpr_mu_lock(&s->mu);
   GPR_ASSERT(s->active_ports == 0);
   s->pollsets = pollsets;
+  s->grpc_server = server;
   for (i = 0; i < s->nports; i++) {
     for (j = 0; j < pollset_count; j++) {
       grpc_pollset_add_fd(exec_ctx, pollsets[j], s->ports[i].emfd);
diff --git a/src/core/iomgr/udp_server.h b/src/core/iomgr/udp_server.h
index 76082d7761..8e3abae864 100644
--- a/src/core/iomgr/udp_server.h
+++ b/src/core/iomgr/udp_server.h
@@ -36,18 +36,22 @@
 
 #include "src/core/iomgr/endpoint.h"
 
+/* Forward decl of grpc_server */
+typedef struct grpc_server grpc_server;
+
 /* Forward decl of grpc_udp_server */
 typedef struct grpc_udp_server grpc_udp_server;
 
 /* Called when data is available to read from the socket. */
-typedef void (*grpc_udp_server_read_cb)(int fd);
+typedef void (*grpc_udp_server_read_cb)(int fd, grpc_server *server);
 
 /* Create a server, initially not bound to any ports */
 grpc_udp_server *grpc_udp_server_create(void);
 
 /* Start listening to bound ports */
 void grpc_udp_server_start(grpc_exec_ctx *exec_ctx, grpc_udp_server *udp_server,
-                           grpc_pollset **pollsets, size_t pollset_count);
+                           grpc_pollset **pollsets, size_t pollset_count,
+                           grpc_server *server);
 
 int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned index);
 
diff --git a/src/core/security/client_auth_filter.c b/src/core/security/client_auth_filter.c
index c152a06498..3f4c084ffc 100644
--- a/src/core/security/client_auth_filter.c
+++ b/src/core/security/client_auth_filter.c
@@ -326,12 +326,10 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
       (grpc_channel_security_connector *)GRPC_SECURITY_CONNECTOR_REF(
           sc, "client_auth_filter");
   chand->md_ctx = metadata_context;
-  chand->authority_string =
-      grpc_mdstr_from_string(chand->md_ctx, ":authority", 0);
-  chand->path_string = grpc_mdstr_from_string(chand->md_ctx, ":path", 0);
-  chand->error_msg_key =
-      grpc_mdstr_from_string(chand->md_ctx, "grpc-message", 0);
-  chand->status_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-status", 0);
+  chand->authority_string = grpc_mdstr_from_string(chand->md_ctx, ":authority");
+  chand->path_string = grpc_mdstr_from_string(chand->md_ctx, ":path");
+  chand->error_msg_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-message");
+  chand->status_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-status");
 }
 
 /* Destructor for channel data */
diff --git a/src/core/support/block_annotate.h b/src/core/support/block_annotate.h
new file mode 100644
index 0000000000..3cd8eee272
--- /dev/null
+++ b/src/core/support/block_annotate.h
@@ -0,0 +1,48 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_INTERNAL_CORE_SUPPORT_BLOCK_ANNOTATE_H
+#define GRPC_INTERNAL_CORE_SUPPORT_BLOCK_ANNOTATE_H
+
+/* These annotations identify the beginning and end of regions where
+   the code may block for reasons other than synchronization functions.
+   These include poll, epoll, and getaddrinfo. */
+
+#define GRPC_SCHEDULING_START_BLOCKING_REGION \
+  do {                                        \
+  } while (0)
+#define GRPC_SCHEDULING_END_BLOCKING_REGION \
+  do {                                      \
+  } while (0)
+
+#endif /* GRPC_INTERNAL_CORE_SUPPORT_BLOCK_ANNOTATE_H */
diff --git a/src/core/support/file.c b/src/core/support/file.c
index c1361d8a9e..8c673dbcc6 100644
--- a/src/core/support/file.c
+++ b/src/core/support/file.c
@@ -40,6 +40,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+#include "src/core/support/block_annotate.h"
 #include "src/core/support/string.h"
 
 gpr_slice gpr_load_file(const char *filename, int add_null_terminator,
@@ -48,9 +49,11 @@ gpr_slice gpr_load_file(const char *filename, int add_null_terminator,
   size_t contents_size = 0;
   char *error_msg = NULL;
   gpr_slice result = gpr_empty_slice();
-  FILE *file = fopen(filename, "rb");
+  FILE *file;
   size_t bytes_read = 0;
 
+  GRPC_SCHEDULING_START_BLOCKING_REGION;
+  file = fopen(filename, "rb");
   if (file == NULL) {
     gpr_asprintf(&error_msg, "Could not open file %s (error = %s).", filename,
                  strerror(errno));
@@ -83,5 +86,6 @@ end:
     if (success != NULL) *success = 0;
   }
   if (file != NULL) fclose(file);
+  GRPC_SCHEDULING_END_BLOCKING_REGION;
   return result;
 }
diff --git a/src/core/support/slice_buffer.c b/src/core/support/slice_buffer.c
index 8873d459d5..a1aa56fd72 100644
--- a/src/core/support/slice_buffer.c
+++ b/src/core/support/slice_buffer.c
@@ -208,7 +208,7 @@ void gpr_slice_buffer_move_into(gpr_slice_buffer *src, gpr_slice_buffer *dst) {
   src->length = 0;
 }
 
-void gpr_slice_buffer_trim_end(gpr_slice_buffer *sb, size_t n) {
+void gpr_slice_buffer_trim_end(gpr_slice_buffer *sb, size_t n, gpr_slice_buffer *garbage) {
   GPR_ASSERT(n <= sb->length);
   sb->length -= n;
   for (;;) {
@@ -216,14 +216,15 @@ void gpr_slice_buffer_trim_end(gpr_slice_buffer *sb, size_t n) {
     gpr_slice slice = sb->slices[idx];
     size_t slice_len = GPR_SLICE_LENGTH(slice);
     if (slice_len > n) {
-      sb->slices[idx] = gpr_slice_sub_no_ref(slice, 0, slice_len - n);
+      sb->slices[idx] = gpr_slice_split_head(&slice, slice_len - n);
+      gpr_slice_buffer_add_indexed(garbage, slice);
       return;
     } else if (slice_len == n) {
-      gpr_slice_unref(slice);
+      gpr_slice_buffer_add_indexed(garbage, slice);
       sb->count = idx;
       return;
     } else {
-      gpr_slice_unref(slice);
+      gpr_slice_buffer_add_indexed(garbage, slice);
       n -= slice_len;
       sb->count = idx;
     }
diff --git a/src/core/support/time_posix.c b/src/core/support/time_posix.c
index dcecff0d05..eedfd0a060 100644
--- a/src/core/support/time_posix.c
+++ b/src/core/support/time_posix.c
@@ -41,6 +41,7 @@
 #include <unistd.h>
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
+#include "src/core/support/block_annotate.h"
 
 static struct timespec timespec_from_gpr(gpr_timespec gts) {
   struct timespec rv;
@@ -126,6 +127,7 @@ void gpr_sleep_until(gpr_timespec until) {
   gpr_timespec now;
   gpr_timespec delta;
   struct timespec delta_ts;
+  int ns_result;
 
   for (;;) {
     /* We could simplify by using clock_nanosleep instead, but it might be
@@ -137,7 +139,10 @@ void gpr_sleep_until(gpr_timespec until) {
 
     delta = gpr_time_sub(until, now);
     delta_ts = timespec_from_gpr(delta);
-    if (nanosleep(&delta_ts, NULL) == 0) {
+    GRPC_SCHEDULING_START_BLOCKING_REGION;
+    ns_result = nanosleep(&delta_ts, NULL);
+    GRPC_SCHEDULING_END_BLOCKING_REGION;
+    if (ns_result == 0) {
       break;
     }
   }
diff --git a/src/core/support/time_precise.h b/src/core/support/time_precise.h
index a72d37e2f1..cd201faab9 100644
--- a/src/core/support/time_precise.h
+++ b/src/core/support/time_precise.h
@@ -84,7 +84,7 @@ static void gpr_precise_clock_now(gpr_timespec *clk) {
 }
 
 #else  /* GRPC_TIMERS_RDTSC */
-static void gpr_precise_clock_now(gpr_timespec* clk) {
+static void gpr_precise_clock_now(gpr_timespec *clk) {
   *clk = gpr_now(GPR_CLOCK_REALTIME);
   clk->clock_type = GPR_CLOCK_PRECISE;
 }
diff --git a/src/core/support/time_win32.c b/src/core/support/time_win32.c
index f794855429..bc0586d069 100644
--- a/src/core/support/time_win32.c
+++ b/src/core/support/time_win32.c
@@ -41,6 +41,8 @@
 #include <src/core/support/time_precise.h>
 #include <sys/timeb.h>
 
+#include "src/core/support/block_annotate.h"
+
 static LARGE_INTEGER g_start_time;
 static double g_time_scale;
 
@@ -92,7 +94,9 @@ void gpr_sleep_until(gpr_timespec until) {
     delta = gpr_time_sub(until, now);
     sleep_millis =
         (DWORD)delta.tv_sec * GPR_MS_PER_SEC + delta.tv_nsec / GPR_NS_PER_MS;
+    GRPC_SCHEDULING_START_BLOCKING_REGION;
     Sleep(sleep_millis);
+    GRPC_SCHEDULING_END_BLOCKING_REGION;
   }
 }
 
diff --git a/src/core/surface/call.c b/src/core/surface/call.c
index 51dcbeb1a1..0b917f1561 100644
--- a/src/core/surface/call.c
+++ b/src/core/surface/call.c
@@ -1030,7 +1030,7 @@ static int prepare_application_metadata(grpc_call *call, size_t count,
     GPR_ASSERT(sizeof(grpc_linked_mdelem) == sizeof(md->internal_data));
     l->md = grpc_mdelem_from_string_and_buffer(call->metadata_context, md->key,
                                                (const gpr_uint8 *)md->value,
-                                               md->value_length, 1);
+                                               md->value_length);
     if (!grpc_mdstr_is_legal_header(l->md->key)) {
       gpr_log(GPR_ERROR, "attempt to send invalid metadata key: %s",
               grpc_mdstr_as_c_string(l->md->key));
@@ -1330,7 +1330,7 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c,
 static grpc_call_error cancel_with_status(grpc_call *c, grpc_status_code status,
                                           const char *description) {
   grpc_mdstr *details =
-      description ? grpc_mdstr_from_string(c->metadata_context, description, 0)
+      description ? grpc_mdstr_from_string(c->metadata_context, description)
                   : NULL;
 
   GPR_ASSERT(status != GRPC_STATUS_OK);
@@ -1689,7 +1689,7 @@ grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops,
             op->data.send_status_from_server.status_details != NULL
                 ? grpc_mdstr_from_string(
                       call->metadata_context,
-                      op->data.send_status_from_server.status_details, 0)
+                      op->data.send_status_from_server.status_details)
                 : NULL;
         req = &reqs[out++];
         if (out > GRPC_IOREQ_OP_COUNT) {
diff --git a/src/core/surface/channel.c b/src/core/surface/channel.c
index 08d003daca..aed7a79e4e 100644
--- a/src/core/surface/channel.c
+++ b/src/core/surface/channel.c
@@ -104,22 +104,21 @@ grpc_channel *grpc_channel_create_from_filters(
   /* decremented by grpc_channel_destroy */
   gpr_ref_init(&channel->refs, 1);
   channel->metadata_context = mdctx;
-  channel->grpc_status_string = grpc_mdstr_from_string(mdctx, "grpc-status", 0);
+  channel->grpc_status_string = grpc_mdstr_from_string(mdctx, "grpc-status");
   channel->grpc_compression_algorithm_string =
-      grpc_mdstr_from_string(mdctx, "grpc-encoding", 0);
+      grpc_mdstr_from_string(mdctx, "grpc-encoding");
   channel->grpc_encodings_accepted_by_peer_string =
-      grpc_mdstr_from_string(mdctx, "grpc-accept-encoding", 0);
-  channel->grpc_message_string =
-      grpc_mdstr_from_string(mdctx, "grpc-message", 0);
+      grpc_mdstr_from_string(mdctx, "grpc-accept-encoding");
+  channel->grpc_message_string = grpc_mdstr_from_string(mdctx, "grpc-message");
   for (i = 0; i < NUM_CACHED_STATUS_ELEMS; i++) {
     char buf[GPR_LTOA_MIN_BUFSIZE];
     gpr_ltoa((long)i, buf);
     channel->grpc_status_elem[i] = grpc_mdelem_from_metadata_strings(
         mdctx, GRPC_MDSTR_REF(channel->grpc_status_string),
-        grpc_mdstr_from_string(mdctx, buf, 0));
+        grpc_mdstr_from_string(mdctx, buf));
   }
-  channel->path_string = grpc_mdstr_from_string(mdctx, ":path", 0);
-  channel->authority_string = grpc_mdstr_from_string(mdctx, ":authority", 0);
+  channel->path_string = grpc_mdstr_from_string(mdctx, ":path");
+  channel->authority_string = grpc_mdstr_from_string(mdctx, ":authority");
   gpr_mu_init(&channel->registered_call_mu);
   channel->registered_calls = NULL;
 
@@ -219,11 +218,11 @@ grpc_call *grpc_channel_create_call(grpc_channel *channel,
       channel, parent_call, propagation_mask, cq,
       grpc_mdelem_from_metadata_strings(
           channel->metadata_context, GRPC_MDSTR_REF(channel->path_string),
-          grpc_mdstr_from_string(channel->metadata_context, method, 0)),
+          grpc_mdstr_from_string(channel->metadata_context, method)),
       host ? grpc_mdelem_from_metadata_strings(
                  channel->metadata_context,
                  GRPC_MDSTR_REF(channel->authority_string),
-                 grpc_mdstr_from_string(channel->metadata_context, host, 0))
+                 grpc_mdstr_from_string(channel->metadata_context, host))
            : NULL,
       deadline);
 }
@@ -234,12 +233,12 @@ void *grpc_channel_register_call(grpc_channel *channel, const char *method,
   GPR_ASSERT(!reserved);
   rc->path = grpc_mdelem_from_metadata_strings(
       channel->metadata_context, GRPC_MDSTR_REF(channel->path_string),
-      grpc_mdstr_from_string(channel->metadata_context, method, 0));
+      grpc_mdstr_from_string(channel->metadata_context, method));
   rc->authority =
       host ? grpc_mdelem_from_metadata_strings(
                  channel->metadata_context,
                  GRPC_MDSTR_REF(channel->authority_string),
-                 grpc_mdstr_from_string(channel->metadata_context, host, 0))
+                 grpc_mdstr_from_string(channel->metadata_context, host))
            : NULL;
   gpr_mu_lock(&channel->registered_call_mu);
   rc->next = channel->registered_calls;
@@ -358,7 +357,7 @@ grpc_mdelem *grpc_channel_get_reffed_status_elem(grpc_channel *channel, int i) {
     gpr_ltoa(i, tmp);
     return grpc_mdelem_from_metadata_strings(
         channel->metadata_context, GRPC_MDSTR_REF(channel->grpc_status_string),
-        grpc_mdstr_from_string(channel->metadata_context, tmp, 0));
+        grpc_mdstr_from_string(channel->metadata_context, tmp));
   }
 }
 
diff --git a/src/core/surface/server.c b/src/core/surface/server.c
index 27fc0945f9..41191ebf26 100644
--- a/src/core/surface/server.c
+++ b/src/core/surface/server.c
@@ -729,9 +729,8 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
   GPR_ASSERT(!is_last);
   chand->server = NULL;
   chand->channel = NULL;
-  chand->path_key = grpc_mdstr_from_string(metadata_context, ":path", 0);
-  chand->authority_key =
-      grpc_mdstr_from_string(metadata_context, ":authority", 0);
+  chand->path_key = grpc_mdstr_from_string(metadata_context, ":path");
+  chand->authority_key = grpc_mdstr_from_string(metadata_context, ":authority");
   chand->next = chand->prev = chand;
   chand->registered_methods = NULL;
   chand->connectivity_state = GRPC_CHANNEL_IDLE;
@@ -959,8 +958,8 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
     chand->registered_methods = gpr_malloc(alloc);
     memset(chand->registered_methods, 0, alloc);
     for (rm = s->registered_methods; rm; rm = rm->next) {
-      host = rm->host ? grpc_mdstr_from_string(mdctx, rm->host, 0) : NULL;
-      method = grpc_mdstr_from_string(mdctx, rm->method, 0);
+      host = rm->host ? grpc_mdstr_from_string(mdctx, rm->host) : NULL;
+      method = grpc_mdstr_from_string(mdctx, rm->method);
       hash = GRPC_MDSTR_KV_HASH(host ? host->hash : 0, method->hash);
       for (probes = 0; chand->registered_methods[(hash + probes) % slots]
                            .server_registered_method != NULL;
diff --git a/src/core/transport/chttp2/incoming_metadata.c b/src/core/transport/chttp2/incoming_metadata.c
index d216c42113..10c64f3356 100644
--- a/src/core/transport/chttp2/incoming_metadata.c
+++ b/src/core/transport/chttp2/incoming_metadata.c
@@ -171,7 +171,7 @@ void grpc_chttp2_incoming_metadata_buffer_postprocess_sopb_and_begin_live_op(
       size_t copy_bytes = sizeof(*buffer->elems) * new_count;
       GPR_ASSERT(mdidx < buffer->count);
       buffer->elems = gpr_malloc(copy_bytes);
-      memcpy(live_op_buffer->elems + mdidx, buffer->elems, copy_bytes);
+      memcpy(buffer->elems, live_op_buffer->elems + mdidx, copy_bytes);
       buffer->count = buffer->capacity = new_count;
     } else {
       buffer->elems = NULL;
diff --git a/src/core/transport/chttp2/stream_encoder.c b/src/core/transport/chttp2/stream_encoder.c
index eb02ccdec3..b1f1db05d2 100644
--- a/src/core/transport/chttp2/stream_encoder.c
+++ b/src/core/transport/chttp2/stream_encoder.c
@@ -466,7 +466,7 @@ static void deadline_enc(grpc_chttp2_hpack_compressor *c, gpr_timespec deadline,
       gpr_time_sub(deadline, gpr_now(deadline.clock_type)), timeout_str);
   mdelem = grpc_mdelem_from_metadata_strings(
       c->mdctx, GRPC_MDSTR_REF(c->timeout_key_str),
-      grpc_mdstr_from_string(c->mdctx, timeout_str, 0));
+      grpc_mdstr_from_string(c->mdctx, timeout_str));
   mdelem = hpack_enc(c, mdelem, st);
   if (mdelem) GRPC_MDELEM_UNREF(mdelem);
 }
@@ -481,7 +481,7 @@ void grpc_chttp2_hpack_compressor_init(grpc_chttp2_hpack_compressor *c,
                                        grpc_mdctx *ctx) {
   memset(c, 0, sizeof(*c));
   c->mdctx = ctx;
-  c->timeout_key_str = grpc_mdstr_from_string(ctx, "grpc-timeout", 0);
+  c->timeout_key_str = grpc_mdstr_from_string(ctx, "grpc-timeout");
 }
 
 void grpc_chttp2_hpack_compressor_destroy(grpc_chttp2_hpack_compressor *c) {
diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c
index 1679e4345d..0202d53abd 100644
--- a/src/core/transport/chttp2_transport.c
+++ b/src/core/transport/chttp2_transport.c
@@ -243,7 +243,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   t->global.pings.next = t->global.pings.prev = &t->global.pings;
   t->parsing.is_client = is_client;
   t->parsing.str_grpc_timeout =
-      grpc_mdstr_from_string(t->metadata_context, "grpc-timeout", 0);
+      grpc_mdstr_from_string(t->metadata_context, "grpc-timeout");
   t->parsing.deframe_state =
       is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0;
   t->writing.is_client = is_client;
diff --git a/src/core/transport/metadata.c b/src/core/transport/metadata.c
index 9d135f4356..3dbb9f0b53 100644
--- a/src/core/transport/metadata.c
+++ b/src/core/transport/metadata.c
@@ -312,38 +312,7 @@ static void slice_unref(void *p) {
   unlock(ctx);
 }
 
-grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str,
-                                   int canonicalize_key) {
-  if (canonicalize_key) {
-    size_t len;
-    size_t i;
-    int canonical = 1;
-
-    for (i = 0; str[i]; i++) {
-      if (str[i] >= 'A' && str[i] <= 'Z') {
-        canonical = 0;
-        /* Keep going in loop just to get string length */
-      }
-    }
-    len = i;
-
-    if (canonical) {
-      return grpc_mdstr_from_buffer(ctx, (const gpr_uint8 *)str, len);
-    } else {
-      char *copy = gpr_malloc(len);
-      grpc_mdstr *ret;
-      for (i = 0; i < len; i++) {
-        if (str[i] >= 'A' && str[i] <= 'Z') {
-          copy[i] = (char)(str[i] - 'A' + 'a');
-        } else {
-          copy[i] = str[i];
-        }
-      }
-      ret = grpc_mdstr_from_buffer(ctx, (const gpr_uint8 *)copy, len);
-      gpr_free(copy);
-      return ret;
-    }
-  }
+grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str) {
   return grpc_mdstr_from_buffer(ctx, (const gpr_uint8 *)str, strlen(str));
 }
 
@@ -524,9 +493,9 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdctx *ctx,
 
 grpc_mdelem *grpc_mdelem_from_strings(grpc_mdctx *ctx, const char *key,
                                       const char *value) {
-  return grpc_mdelem_from_metadata_strings(
-      ctx, grpc_mdstr_from_string(ctx, key, 0),
-      grpc_mdstr_from_string(ctx, value, 0));
+  return grpc_mdelem_from_metadata_strings(ctx,
+                                           grpc_mdstr_from_string(ctx, key),
+                                           grpc_mdstr_from_string(ctx, value));
 }
 
 grpc_mdelem *grpc_mdelem_from_slices(grpc_mdctx *ctx, gpr_slice key,
@@ -538,10 +507,9 @@ grpc_mdelem *grpc_mdelem_from_slices(grpc_mdctx *ctx, gpr_slice key,
 grpc_mdelem *grpc_mdelem_from_string_and_buffer(grpc_mdctx *ctx,
                                                 const char *key,
                                                 const gpr_uint8 *value,
-                                                size_t value_length,
-                                                int canonicalize_key) {
+                                                size_t value_length) {
   return grpc_mdelem_from_metadata_strings(
-      ctx, grpc_mdstr_from_string(ctx, key, canonicalize_key),
+      ctx, grpc_mdstr_from_string(ctx, key),
       grpc_mdstr_from_buffer(ctx, value, value_length));
 }
 
diff --git a/src/core/transport/metadata.h b/src/core/transport/metadata.h
index eb17747be7..136a65f288 100644
--- a/src/core/transport/metadata.h
+++ b/src/core/transport/metadata.h
@@ -95,8 +95,7 @@ size_t grpc_mdctx_get_mdtab_free_test_only(grpc_mdctx *mdctx);
 
 /* Constructors for grpc_mdstr instances; take a variety of data types that
    clients may have handy */
-grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str,
-                                   int perform_key_canonicalization);
+grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str);
 /* Unrefs the slice. */
 grpc_mdstr *grpc_mdstr_from_slice(grpc_mdctx *ctx, gpr_slice slice);
 grpc_mdstr *grpc_mdstr_from_buffer(grpc_mdctx *ctx, const gpr_uint8 *str,
@@ -118,8 +117,7 @@ grpc_mdelem *grpc_mdelem_from_slices(grpc_mdctx *ctx, gpr_slice key,
 grpc_mdelem *grpc_mdelem_from_string_and_buffer(grpc_mdctx *ctx,
                                                 const char *key,
                                                 const gpr_uint8 *value,
-                                                size_t value_length,
-                                                int canonicalize_key);
+                                                size_t value_length);
 
 /* Mutator and accessor for grpc_mdelem user data. The destructor function
    is used as a type tag and is checked during user_data fetch. */