Merge branch 'master' of https://github.com/grpc/grpc into no-more-extern-c

16 files changed, 287 insertions, 58 deletions

diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
index 5bd647885f..01a16955d9 100644
--- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
+++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
@@ -1758,7 +1758,7 @@ static void send_goaway(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
   grpc_http2_error_code http_error;
   grpc_slice slice;
   grpc_error_get_status(exec_ctx, error, GRPC_MILLIS_INF_FUTURE, nullptr,
-                        &slice, &http_error);
+                        &slice, &http_error, nullptr);
   grpc_chttp2_goaway_append(t->last_new_stream_id, (uint32_t)http_error,
                             grpc_slice_ref_internal(slice), &t->qbuf);
   grpc_chttp2_initiate_write(exec_ctx, t,
@@ -2061,7 +2061,7 @@ void grpc_chttp2_cancel_stream(grpc_exec_ctx* exec_ctx,
     if (s->id != 0) {
       grpc_http2_error_code http_error;
       grpc_error_get_status(exec_ctx, due_to_error, s->deadline, nullptr,
-                            nullptr, &http_error);
+                            nullptr, &http_error, nullptr);
       grpc_slice_buffer_add(
           &t->qbuf, grpc_chttp2_rst_stream_create(s->id, (uint32_t)http_error,
                                                   &s->stats.outgoing));
@@ -2079,7 +2079,8 @@ void grpc_chttp2_fake_status(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
                              grpc_chttp2_stream* s, grpc_error* error) {
   grpc_status_code status;
   grpc_slice slice;
-  grpc_error_get_status(exec_ctx, error, s->deadline, &status, &slice, nullptr);
+  grpc_error_get_status(exec_ctx, error, s->deadline, &status, &slice, nullptr,
+                        nullptr);
   if (status != GRPC_STATUS_OK) {
     s->seen_error = true;
   }
@@ -2244,7 +2245,7 @@ static void close_from_api(grpc_exec_ctx* exec_ctx, grpc_chttp2_transport* t,
   grpc_status_code grpc_status;
   grpc_slice slice;
   grpc_error_get_status(exec_ctx, error, s->deadline, &grpc_status, &slice,
-                        nullptr);
+                        nullptr, nullptr);
 
   GPR_ASSERT(grpc_status >= 0 && (int)grpc_status < 100);
 
diff --git a/src/core/ext/transport/chttp2/transport/flow_control.h b/src/core/ext/transport/chttp2/transport/flow_control.h
index bb710fef83..2515c94309 100644
--- a/src/core/ext/transport/chttp2/transport/flow_control.h
+++ b/src/core/ext/transport/chttp2/transport/flow_control.h
@@ -19,6 +19,7 @@
 #ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FLOW_CONTROL_H
 #define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FLOW_CONTROL_H
 
+#include <grpc/support/port_platform.h>
 #include <stdint.h>
 
 #include <grpc/support/useful.h>
diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h
index 93b5625aad..1b6e5396a5 100644
--- a/src/core/lib/channel/channel_stack.h
+++ b/src/core/lib/channel/channel_stack.h
@@ -80,6 +80,7 @@ typedef struct {
 typedef struct {
   grpc_call_stats stats;
   grpc_status_code final_status;
+  const char** error_string;
 } grpc_call_final_info;
 
 /* Channel filters specify:
diff --git a/src/core/lib/iomgr/ev_epoll1_linux.cc b/src/core/lib/iomgr/ev_epoll1_linux.cc
index 60446f7dd5..0dda1d924c 100644
--- a/src/core/lib/iomgr/ev_epoll1_linux.cc
+++ b/src/core/lib/iomgr/ev_epoll1_linux.cc
@@ -263,11 +263,13 @@ static grpc_fd* fd_create(int fd, const char* name) {
 
   if (new_fd == nullptr) {
     new_fd = (grpc_fd*)gpr_malloc(sizeof(grpc_fd));
+    new_fd->read_closure.Init();
+    new_fd->write_closure.Init();
   }
 
   new_fd->fd = fd;
-  new_fd->read_closure.Init();
-  new_fd->write_closure.Init();
+  new_fd->read_closure->InitEvent();
+  new_fd->write_closure->InitEvent();
   gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL);
 
   new_fd->freelist_next = nullptr;
@@ -336,8 +338,8 @@ static void fd_orphan(grpc_exec_ctx* exec_ctx, grpc_fd* fd,
   GRPC_CLOSURE_SCHED(exec_ctx, on_done, GRPC_ERROR_REF(error));
 
   grpc_iomgr_unregister_object(&fd->iomgr_object);
-  fd->read_closure.Destroy();
-  fd->write_closure.Destroy();
+  fd->read_closure->DestroyEvent();
+  fd->write_closure->DestroyEvent();
 
   gpr_mu_lock(&fd_freelist_mu);
   fd->freelist_next = fd_freelist;
diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc
index 10b84401fa..62643df697 100644
--- a/src/core/lib/iomgr/ev_epollex_linux.cc
+++ b/src/core/lib/iomgr/ev_epollex_linux.cc
@@ -286,8 +286,8 @@ static void fd_destroy(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {
   fd->freelist_next = fd_freelist;
   fd_freelist = fd;
 
-  fd->read_closure.Destroy();
-  fd->write_closure.Destroy();
+  fd->read_closure->DestroyEvent();
+  fd->write_closure->DestroyEvent();
 
   gpr_mu_unlock(&fd_freelist_mu);
 }
@@ -340,6 +340,8 @@ static grpc_fd* fd_create(int fd, const char* name) {
 
   if (new_fd == nullptr) {
     new_fd = (grpc_fd*)gpr_malloc(sizeof(grpc_fd));
+    new_fd->read_closure.Init();
+    new_fd->write_closure.Init();
   }
 
   gpr_mu_init(&new_fd->pollable_mu);
@@ -347,8 +349,8 @@ static grpc_fd* fd_create(int fd, const char* name) {
   new_fd->pollable_obj = nullptr;
   gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1);
   new_fd->fd = fd;
-  new_fd->read_closure.Init();
-  new_fd->write_closure.Init();
+  new_fd->read_closure->InitEvent();
+  new_fd->write_closure->InitEvent();
   gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL);
 
   new_fd->freelist_next = nullptr;
diff --git a/src/core/lib/iomgr/ev_epollsig_linux.cc b/src/core/lib/iomgr/ev_epollsig_linux.cc
index 689c0d4573..12c8483b8e 100644
--- a/src/core/lib/iomgr/ev_epollsig_linux.cc
+++ b/src/core/lib/iomgr/ev_epollsig_linux.cc
@@ -767,8 +767,8 @@ static void unref_by(grpc_fd* fd, int n) {
     fd_freelist = fd;
     grpc_iomgr_unregister_object(&fd->iomgr_object);
 
-    fd->read_closure.Destroy();
-    fd->write_closure.Destroy();
+    fd->read_closure->DestroyEvent();
+    fd->write_closure->DestroyEvent();
 
     gpr_mu_unlock(&fd_freelist_mu);
   } else {
@@ -819,6 +819,8 @@ static grpc_fd* fd_create(int fd, const char* name) {
   if (new_fd == nullptr) {
     new_fd = (grpc_fd*)gpr_malloc(sizeof(grpc_fd));
     gpr_mu_init(&new_fd->po.mu);
+    new_fd->read_closure.Init();
+    new_fd->write_closure.Init();
   }
 
   /* Note: It is not really needed to get the new_fd->po.mu lock here. If this
@@ -833,8 +835,8 @@ static grpc_fd* fd_create(int fd, const char* name) {
   gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1);
   new_fd->fd = fd;
   new_fd->orphaned = false;
-  new_fd->read_closure.Init();
-  new_fd->write_closure.Init();
+  new_fd->read_closure->InitEvent();
+  new_fd->write_closure->InitEvent();
   gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL);
 
   new_fd->freelist_next = nullptr;
diff --git a/src/core/lib/iomgr/lockfree_event.cc b/src/core/lib/iomgr/lockfree_event.cc
index d477f64ba5..f0e798e8d8 100644
--- a/src/core/lib/iomgr/lockfree_event.cc
+++ b/src/core/lib/iomgr/lockfree_event.cc
@@ -57,7 +57,9 @@ extern grpc_core::TraceFlag grpc_polling_trace;
 
 namespace grpc_core {
 
-LockfreeEvent::LockfreeEvent() {
+LockfreeEvent::LockfreeEvent() { InitEvent(); }
+
+void LockfreeEvent::InitEvent() {
   /* Perform an atomic store to start the state machine.
 
      Note carefully that LockfreeEvent *MAY* be used whilst in a destroyed
@@ -67,7 +69,7 @@ LockfreeEvent::LockfreeEvent() {
   gpr_atm_no_barrier_store(&state_, kClosureNotReady);
 }
 
-LockfreeEvent::~LockfreeEvent() {
+void LockfreeEvent::DestroyEvent() {
   gpr_atm curr;
   do {
     curr = gpr_atm_no_barrier_load(&state_);
diff --git a/src/core/lib/iomgr/lockfree_event.h b/src/core/lib/iomgr/lockfree_event.h
index c667dcd3bc..aec67a3399 100644
--- a/src/core/lib/iomgr/lockfree_event.h
+++ b/src/core/lib/iomgr/lockfree_event.h
@@ -30,11 +30,16 @@ namespace grpc_core {
 class LockfreeEvent {
  public:
   LockfreeEvent();
-  ~LockfreeEvent();
 
   LockfreeEvent(const LockfreeEvent&) = delete;
   LockfreeEvent& operator=(const LockfreeEvent&) = delete;
 
+  // These methods are used to initialize and destroy the internal state. These
+  // cannot be done in constructor and destructor because SetReady may be called
+  // when the event is destroyed and put in a freelist.
+  void InitEvent();
+  void DestroyEvent();
+
   bool IsShutdown() const {
     return (gpr_atm_no_barrier_load(&state_) & kShutdownBit) != 0;
   }
diff --git a/src/core/lib/iomgr/tcp_posix.cc b/src/core/lib/iomgr/tcp_posix.cc
index 6d9e044fa2..d09cfca9af 100644
--- a/src/core/lib/iomgr/tcp_posix.cc
+++ b/src/core/lib/iomgr/tcp_posix.cc
@@ -81,9 +81,7 @@ typedef struct {
 
   grpc_slice_buffer* incoming_buffer;
   grpc_slice_buffer* outgoing_buffer;
-  /** slice within outgoing_buffer to write next */
-  size_t outgoing_slice_idx;
-  /** byte within outgoing_buffer->slices[outgoing_slice_idx] to write next */
+  /** byte within outgoing_buffer->slices[0] to write next */
   size_t outgoing_byte_idx;
 
   grpc_closure* read_cb;
@@ -532,23 +530,26 @@ static bool tcp_flush(grpc_exec_ctx* exec_ctx, grpc_tcp* tcp,
   size_t unwind_slice_idx;
   size_t unwind_byte_idx;
 
+  // We always start at zero, because we eagerly unref and trim the slice
+  // buffer as we write
+  size_t outgoing_slice_idx = 0;
+
   for (;;) {
     sending_length = 0;
-    unwind_slice_idx = tcp->outgoing_slice_idx;
+    unwind_slice_idx = outgoing_slice_idx;
     unwind_byte_idx = tcp->outgoing_byte_idx;
-    for (iov_size = 0; tcp->outgoing_slice_idx != tcp->outgoing_buffer->count &&
+    for (iov_size = 0; outgoing_slice_idx != tcp->outgoing_buffer->count &&
                        iov_size != MAX_WRITE_IOVEC;
          iov_size++) {
       iov[iov_size].iov_base =
           GRPC_SLICE_START_PTR(
-              tcp->outgoing_buffer->slices[tcp->outgoing_slice_idx]) +
+              tcp->outgoing_buffer->slices[outgoing_slice_idx]) +
           tcp->outgoing_byte_idx;
       iov[iov_size].iov_len =
-          GRPC_SLICE_LENGTH(
-              tcp->outgoing_buffer->slices[tcp->outgoing_slice_idx]) -
+          GRPC_SLICE_LENGTH(tcp->outgoing_buffer->slices[outgoing_slice_idx]) -
           tcp->outgoing_byte_idx;
       sending_length += iov[iov_size].iov_len;
-      tcp->outgoing_slice_idx++;
+      outgoing_slice_idx++;
       tcp->outgoing_byte_idx = 0;
     }
     GPR_ASSERT(iov_size > 0);
@@ -574,16 +575,25 @@ static bool tcp_flush(grpc_exec_ctx* exec_ctx, grpc_tcp* tcp,
 
     if (sent_length < 0) {
       if (errno == EAGAIN) {
-        tcp->outgoing_slice_idx = unwind_slice_idx;
         tcp->outgoing_byte_idx = unwind_byte_idx;
+        // unref all and forget about all slices that have been written to this
+        // point
+        for (size_t idx = 0; idx < unwind_slice_idx; ++idx) {
+          grpc_slice_unref_internal(
+              exec_ctx, grpc_slice_buffer_take_first(tcp->outgoing_buffer));
+        }
         return false;
       } else if (errno == EPIPE) {
         *error = grpc_error_set_int(GRPC_OS_ERROR(errno, "sendmsg"),
                                     GRPC_ERROR_INT_GRPC_STATUS,
                                     GRPC_STATUS_UNAVAILABLE);
+        grpc_slice_buffer_reset_and_unref_internal(exec_ctx,
+                                                   tcp->outgoing_buffer);
         return true;
       } else {
         *error = tcp_annotate_error(GRPC_OS_ERROR(errno, "sendmsg"), tcp);
+        grpc_slice_buffer_reset_and_unref_internal(exec_ctx,
+                                                   tcp->outgoing_buffer);
         return true;
       }
     }
@@ -593,9 +603,9 @@ static bool tcp_flush(grpc_exec_ctx* exec_ctx, grpc_tcp* tcp,
     while (trailing > 0) {
       size_t slice_length;
 
-      tcp->outgoing_slice_idx--;
-      slice_length = GRPC_SLICE_LENGTH(
-          tcp->outgoing_buffer->slices[tcp->outgoing_slice_idx]);
+      outgoing_slice_idx--;
+      slice_length =
+          GRPC_SLICE_LENGTH(tcp->outgoing_buffer->slices[outgoing_slice_idx]);
       if (slice_length > trailing) {
         tcp->outgoing_byte_idx = slice_length - trailing;
         break;
@@ -604,11 +614,13 @@ static bool tcp_flush(grpc_exec_ctx* exec_ctx, grpc_tcp* tcp,
       }
     }
 
-    if (tcp->outgoing_slice_idx == tcp->outgoing_buffer->count) {
+    if (outgoing_slice_idx == tcp->outgoing_buffer->count) {
       *error = GRPC_ERROR_NONE;
+      grpc_slice_buffer_reset_and_unref_internal(exec_ctx,
+                                                 tcp->outgoing_buffer);
       return true;
     }
-  };
+  }
 }
 
 static void tcp_handle_write(grpc_exec_ctx* exec_ctx, void* arg /* grpc_tcp */,
@@ -672,7 +684,6 @@ static void tcp_write(grpc_exec_ctx* exec_ctx, grpc_endpoint* ep,
     return;
   }
   tcp->outgoing_buffer = buf;
-  tcp->outgoing_slice_idx = 0;
   tcp->outgoing_byte_idx = 0;
 
   if (!tcp_flush(exec_ctx, tcp, &error)) {
diff --git a/src/core/lib/slice/slice_internal.h b/src/core/lib/slice/slice_internal.h
index 556e6b84b2..ed0070d375 100644
--- a/src/core/lib/slice/slice_internal.h
+++ b/src/core/lib/slice/slice_internal.h
@@ -28,6 +28,9 @@ grpc_slice grpc_slice_ref_internal(grpc_slice slice);
 void grpc_slice_unref_internal(grpc_exec_ctx* exec_ctx, grpc_slice slice);
 void grpc_slice_buffer_reset_and_unref_internal(grpc_exec_ctx* exec_ctx,
                                                 grpc_slice_buffer* sb);
+void grpc_slice_buffer_partial_unref_internal(grpc_exec_ctx* exec_ctx,
+                                              grpc_slice_buffer* sb,
+                                              size_t idx);
 void grpc_slice_buffer_destroy_internal(grpc_exec_ctx* exec_ctx,
                                         grpc_slice_buffer* sb);
 
diff --git a/src/core/lib/support/abstract.h b/src/core/lib/support/abstract.h
new file mode 100644
index 0000000000..5498769a7d
--- /dev/null
+++ b/src/core/lib/support/abstract.h
@@ -0,0 +1,29 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SUPPORT_ABSTRACT_H
+#define GRPC_CORE_LIB_SUPPORT_ABSTRACT_H
+
+// This is needed to support abstract base classes in the c core. Since gRPC
+// doesn't have a c++ runtime, it will hit a linker error on delete unless
+// we define a virtual operator delete. See this blog for more info:
+// https://eli.thegreenplace.net/2015/c-deleting-destructors-and-virtual-operator-delete/
+#define GRPC_ABSTRACT_BASE_CLASS \
+  static void operator delete(void* p) { abort(); }
+
+#endif /* GRPC_CORE_LIB_SUPPORT_ABSTRACT_H */
diff --git a/src/core/lib/support/cpu_posix.cc b/src/core/lib/support/cpu_posix.cc
index 503a96b4c8..bca14a0c12 100644
--- a/src/core/lib/support/cpu_posix.cc
+++ b/src/core/lib/support/cpu_posix.cc
@@ -18,21 +18,23 @@
 
 #include <grpc/support/port_platform.h>
 
-#ifdef GPR_CPU_POSIX
+#if defined(GPR_CPU_POSIX)
 
 #include <errno.h>
+#include <pthread.h>
 #include <string.h>
 #include <unistd.h>
 
+#include <grpc/support/alloc.h>
 #include <grpc/support/cpu.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/useful.h>
 
-static __thread char magic_thread_local;
-
 static long ncpus = 0;
 
+static pthread_key_t thread_id_key;
+
 static void init_ncpus() {
   ncpus = sysconf(_SC_NPROCESSORS_ONLN);
   if (ncpus < 1 || ncpus > INT32_MAX) {
@@ -47,12 +49,32 @@ unsigned gpr_cpu_num_cores(void) {
   return (unsigned)ncpus;
 }
 
+static void delete_thread_id(void* value) {
+  if (value) {
+    gpr_free(value);
+  }
+}
+
+static void init_thread_id_key(void) {
+  pthread_key_create(&thread_id_key, delete_thread_id);
+}
+
 unsigned gpr_cpu_current_cpu(void) {
   /* NOTE: there's no way I know to return the actual cpu index portably...
      most code that's using this is using it to shard across work queues though,
      so here we use thread identity instead to achieve a similar though not
      identical effect */
-  return (unsigned)GPR_HASH_POINTER(&magic_thread_local, gpr_cpu_num_cores());
+  static gpr_once once = GPR_ONCE_INIT;
+  gpr_once_init(&once, init_thread_id_key);
+
+  unsigned int* thread_id =
+      static_cast<unsigned int*>(pthread_getspecific(thread_id_key));
+  if (thread_id == nullptr) {
+    thread_id = static_cast<unsigned int*>(gpr_malloc(sizeof(unsigned int)));
+    pthread_setspecific(thread_id_key, thread_id);
+  }
+
+  return (unsigned)GPR_HASH_POINTER(thread_id, gpr_cpu_num_cores());
 }
 
 #endif /* GPR_CPU_POSIX */
diff --git a/src/core/lib/support/manual_constructor.h b/src/core/lib/support/manual_constructor.h
index d753cf98a0..fda7653dbc 100644
--- a/src/core/lib/support/manual_constructor.h
+++ b/src/core/lib/support/manual_constructor.h
@@ -22,12 +22,147 @@
 // manually construct a region of memory with some type
 
 #include <stddef.h>
+#include <stdlib.h>
 #include <new>
 #include <type_traits>
 #include <utility>
 
+#include <grpc/support/log.h>
+
 namespace grpc_core {
 
+// this contains templated helpers needed to implement the ManualConstructors
+// in this file.
+namespace manual_ctor_impl {
+
+// is_one_of returns true it a class, Member, is present in a variadic list of
+// classes, List.
+template <class Member, class... List>
+class is_one_of;
+
+template <class Member, class... List>
+class is_one_of<Member, Member, List...> {
+ public:
+  static constexpr const bool value = true;
+};
+
+template <class Member, class A, class... List>
+class is_one_of<Member, A, List...> {
+ public:
+  static constexpr const bool value = is_one_of<Member, List...>::value;
+};
+
+template <class Member>
+class is_one_of<Member> {
+ public:
+  static constexpr const bool value = false;
+};
+
+// max_size_of returns sizeof(Type) for the largest type in the variadic list
+// of classes, Types.
+template <class... Types>
+class max_size_of;
+
+template <class A>
+class max_size_of<A> {
+ public:
+  static constexpr const size_t value = sizeof(A);
+};
+
+template <class A, class... B>
+class max_size_of<A, B...> {
+ public:
+  static constexpr const size_t value = sizeof(A) > max_size_of<B...>::value
+                                            ? sizeof(A)
+                                            : max_size_of<B...>::value;
+};
+
+// max_size_of returns alignof(Type) for the largest type in the variadic list
+// of classes, Types.
+template <class... Types>
+class max_align_of;
+
+template <class A>
+class max_align_of<A> {
+ public:
+  static constexpr const size_t value = alignof(A);
+};
+
+template <class A, class... B>
+class max_align_of<A, B...> {
+ public:
+  static constexpr const size_t value = alignof(A) > max_align_of<B...>::value
+                                            ? alignof(A)
+                                            : max_align_of<B...>::value;
+};
+
+}  // namespace manual_ctor_impl
+
+template <class BaseType, class... DerivedTypes>
+class PolymorphicManualConstructor {
+ public:
+  // No constructor or destructor because one of the most useful uses of
+  // this class is as part of a union, and members of a union could not have
+  // constructors or destructors till C++11.  And, anyway, the whole point of
+  // this class is to bypass constructor and destructor.
+
+  BaseType* get() { return reinterpret_cast<BaseType*>(&space_); }
+  const BaseType* get() const {
+    return reinterpret_cast<const BaseType*>(&space_);
+  }
+
+  BaseType* operator->() { return get(); }
+  const BaseType* operator->() const { return get(); }
+
+  BaseType& operator*() { return *get(); }
+  const BaseType& operator*() const { return *get(); }
+
+  template <class DerivedType>
+  void Init() {
+    FinishInit(new (&space_) DerivedType);
+  }
+
+  // Init() constructs the Type instance using the given arguments
+  // (which are forwarded to Type's constructor).
+  //
+  // Note that Init() with no arguments performs default-initialization,
+  // not zero-initialization (i.e it behaves the same as "new Type;", not
+  // "new Type();"), so it will leave non-class types uninitialized.
+  template <class DerivedType, typename... Ts>
+  void Init(Ts&&... args) {
+    FinishInit(new (&space_) DerivedType(std::forward<Ts>(args)...));
+  }
+
+  // Init() that is equivalent to copy and move construction.
+  // Enables usage like this:
+  //   ManualConstructor<std::vector<int>> v;
+  //   v.Init({1, 2, 3});
+  template <class DerivedType>
+  void Init(const DerivedType& x) {
+    FinishInit(new (&space_) DerivedType(x));
+  }
+  template <class DerivedType>
+  void Init(DerivedType&& x) {
+    FinishInit(new (&space_) DerivedType(std::move(x)));
+  }
+
+  void Destroy() { get()->~BaseType(); }
+
+ private:
+  template <class DerivedType>
+  void FinishInit(DerivedType* p) {
+    static_assert(
+        manual_ctor_impl::is_one_of<DerivedType, DerivedTypes...>::value,
+        "DerivedType must be one of the predeclared DerivedTypes");
+    GPR_ASSERT(reinterpret_cast<BaseType*>(static_cast<DerivedType*>(p)) == p);
+  }
+
+  typename std::aligned_storage<
+      grpc_core::manual_ctor_impl::max_size_of<DerivedTypes...>::value,
+      grpc_core::manual_ctor_impl::max_align_of<DerivedTypes...>::value>::type
+      space_;
+};
+
 template <typename Type>
 class ManualConstructor {
  public:
diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc
index a83c95c8dc..a2eb02bd85 100644
--- a/src/core/lib/surface/call.cc
+++ b/src/core/lib/surface/call.cc
@@ -234,6 +234,7 @@ struct grpc_call {
     struct {
       grpc_status_code* status;
       grpc_slice* status_details;
+      const char** error_string;
     } client;
     struct {
       int* cancelled;
@@ -284,7 +285,8 @@ static void receiving_slice_ready(grpc_exec_ctx* exec_ctx, void* bctlp,
 static void get_final_status(grpc_exec_ctx* exec_ctx, grpc_call* call,
                              void (*set_value)(grpc_status_code code,
                                                void* user_data),
-                             void* set_value_user_data, grpc_slice* details);
+                             void* set_value_user_data, grpc_slice* details,
+                             const char** error_string);
 static void set_status_value_directly(grpc_status_code status, void* dest);
 static void set_status_from_error(grpc_exec_ctx* exec_ctx, grpc_call* call,
                                   status_source source, grpc_error* error);
@@ -546,7 +548,8 @@ static void destroy_call(grpc_exec_ctx* exec_ctx, void* call,
   }
 
   get_final_status(exec_ctx, c, set_status_value_directly,
-                   &c->final_info.final_status, nullptr);
+                   &c->final_info.final_status, nullptr,
+                   c->final_info.error_string);
   c->final_info.stats.latency =
       gpr_time_sub(gpr_now(GPR_CLOCK_MONOTONIC), c->start_time);
 
@@ -733,16 +736,15 @@ static void cancel_with_status(grpc_exec_ctx* exec_ctx, grpc_call* c,
  * FINAL STATUS CODE MANIPULATION
  */
 
-static bool get_final_status_from(grpc_exec_ctx* exec_ctx, grpc_call* call,
-                                  grpc_error* error, bool allow_ok_status,
-                                  void (*set_value)(grpc_status_code code,
-                                                    void* user_data),
-                                  void* set_value_user_data,
-                                  grpc_slice* details) {
+static bool get_final_status_from(
+    grpc_exec_ctx* exec_ctx, grpc_call* call, grpc_error* error,
+    bool allow_ok_status,
+    void (*set_value)(grpc_status_code code, void* user_data),
+    void* set_value_user_data, grpc_slice* details, const char** error_string) {
   grpc_status_code code;
   grpc_slice slice = grpc_empty_slice();
   grpc_error_get_status(exec_ctx, error, call->send_deadline, &code, &slice,
-                        nullptr);
+                        nullptr, error_string);
   if (code == GRPC_STATUS_OK && !allow_ok_status) {
     return false;
   }
@@ -757,7 +759,8 @@ static bool get_final_status_from(grpc_exec_ctx* exec_ctx, grpc_call* call,
 static void get_final_status(grpc_exec_ctx* exec_ctx, grpc_call* call,
                              void (*set_value)(grpc_status_code code,
                                                void* user_data),
-                             void* set_value_user_data, grpc_slice* details) {
+                             void* set_value_user_data, grpc_slice* details,
+                             const char** error_string) {
   int i;
   received_status status[STATUS_SOURCE_COUNT];
   for (i = 0; i < STATUS_SOURCE_COUNT; i++) {
@@ -781,7 +784,7 @@ static void get_final_status(grpc_exec_ctx* exec_ctx, grpc_call* call,
           grpc_error_has_clear_grpc_status(status[i].error)) {
         if (get_final_status_from(exec_ctx, call, status[i].error,
                                   allow_ok_status != 0, set_value,
-                                  set_value_user_data, details)) {
+                                  set_value_user_data, details, error_string)) {
           return;
         }
       }
@@ -791,7 +794,7 @@ static void get_final_status(grpc_exec_ctx* exec_ctx, grpc_call* call,
       if (status[i].is_set) {
         if (get_final_status_from(exec_ctx, call, status[i].error,
                                   allow_ok_status != 0, set_value,
-                                  set_value_user_data, details)) {
+                                  set_value_user_data, details, error_string)) {
           return;
         }
       }
@@ -1332,10 +1335,11 @@ static void post_batch_completion(grpc_exec_ctx* exec_ctx,
     if (call->is_client) {
       get_final_status(exec_ctx, call, set_status_value_directly,
                        call->final_op.client.status,
-                       call->final_op.client.status_details);
+                       call->final_op.client.status_details,
+                       call->final_op.client.error_string);
     } else {
       get_final_status(exec_ctx, call, set_cancelled_value,
-                       call->final_op.server.cancelled, nullptr);
+                       call->final_op.server.cancelled, nullptr, nullptr);
     }
 
     GRPC_ERROR_UNREF(error);
@@ -1992,6 +1996,8 @@ static grpc_call_error call_start_batch(grpc_exec_ctx* exec_ctx,
         call->final_op.client.status = op->data.recv_status_on_client.status;
         call->final_op.client.status_details =
             op->data.recv_status_on_client.status_details;
+        call->final_op.client.error_string =
+            op->data.recv_status_on_client.error_string;
         stream_op->recv_trailing_metadata = true;
         stream_op->collect_stats = true;
         stream_op_payload->recv_trailing_metadata.recv_trailing_metadata =
diff --git a/src/core/lib/transport/error_utils.cc b/src/core/lib/transport/error_utils.cc
index 19510b4c8d..69c8ae6de3 100644
--- a/src/core/lib/transport/error_utils.cc
+++ b/src/core/lib/transport/error_utils.cc
@@ -18,6 +18,7 @@
 
 #include "src/core/lib/transport/error_utils.h"
 
+#include <grpc/support/string_util.h>
 #include "src/core/lib/iomgr/error_internal.h"
 #include "src/core/lib/transport/status_conversion.h"
 
@@ -41,8 +42,8 @@ static grpc_error* recursively_find_error_with_field(grpc_error* error,
 
 void grpc_error_get_status(grpc_exec_ctx* exec_ctx, grpc_error* error,
                            grpc_millis deadline, grpc_status_code* code,
-                           grpc_slice* slice,
-                           grpc_http2_error_code* http_error) {
+                           grpc_slice* slice, grpc_http2_error_code* http_error,
+                           const char** error_string) {
   // Start with the parent error and recurse through the tree of children
   // until we find the first one that has a status code.
   grpc_error* found_error =
@@ -69,6 +70,10 @@ void grpc_error_get_status(grpc_exec_ctx* exec_ctx, grpc_error* error,
   }
   if (code != nullptr) *code = status;
 
+  if (error_string != NULL && status != GRPC_STATUS_OK) {
+    *error_string = gpr_strdup(grpc_error_string(error));
+  }
+
   if (http_error != nullptr) {
     if (grpc_error_get_int(found_error, GRPC_ERROR_INT_HTTP2_ERROR, &integer)) {
       *http_error = (grpc_http2_error_code)integer;
diff --git a/src/core/lib/transport/error_utils.h b/src/core/lib/transport/error_utils.h
index 1c91976ba5..8b006ae992 100644
--- a/src/core/lib/transport/error_utils.h
+++ b/src/core/lib/transport/error_utils.h
@@ -26,13 +26,15 @@
 /// A utility function to get the status code and message to be returned
 /// to the application.  If not set in the top-level message, looks
 /// through child errors until it finds the first one with these attributes.
-/// All attributes are pulled from the same child error. If any of the
-/// attributes (code, msg, http_status) are unneeded, they can be passed as
+/// All attributes are pulled from the same child error. error_string will
+/// be populated with the entire error string. If any of the attributes (code,
+/// msg, http_status, error_string) are unneeded, they can be passed as
 /// NULL.
 void grpc_error_get_status(grpc_exec_ctx* exec_ctx, grpc_error* error,
                            grpc_millis deadline, grpc_status_code* code,
                            grpc_slice* slice,
-                           grpc_http2_error_code* http_status);
+                           grpc_http2_error_code* http_status,
+                           const char** error_string);
 
 /// A utility function to check whether there is a clear status code that
 /// doesn't need to be guessed in \a error. This means that \a error or some