Merge github.com:grpc/grpc into sceq

22 files changed, 857 insertions, 108 deletions

diff --git a/src/core/census/grpc_filter.c b/src/core/census/grpc_filter.c
index a8db32b9d5..c8aaf31e2d 100644
--- a/src/core/census/grpc_filter.c
+++ b/src/core/census/grpc_filter.c
@@ -107,8 +107,8 @@ static void server_mutate_op(grpc_call_element *elem,
   if (op->recv_initial_metadata) {
     /* substitute our callback for the op callback */
     calld->recv_initial_metadata = op->recv_initial_metadata;
-    calld->on_done_recv = op->on_complete;
-    op->on_complete = &calld->finish_recv;
+    calld->on_done_recv = op->recv_initial_metadata_ready;
+    op->recv_initial_metadata_ready = &calld->finish_recv;
   }
 }
 
diff --git a/src/core/census/log.c b/src/core/census/log.c
new file mode 100644
index 0000000000..91b26941b8
--- /dev/null
+++ b/src/core/census/log.c
@@ -0,0 +1,600 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+// Implements an efficient in-memory log, optimized for multiple writers and
+// a single reader. Available log space is divided up in blocks of
+// CENSUS_LOG_2_MAX_RECORD_SIZE bytes. A block can be in one of the following
+// three data structures:
+// - Free blocks (free_block_list)
+// - Blocks with unread data (dirty_block_list)
+// - Blocks currently attached to cores (core_local_blocks[])
+//
+// census_log_start_write() moves a block from core_local_blocks[] to the end of
+// dirty_block_list when block:
+// - is out-of-space OR
+// - has an incomplete record (an incomplete record occurs when a thread calls
+//   census_log_start_write() and is context-switched before calling
+//   census_log_end_write()
+// So, blocks in dirty_block_list are ordered, from oldest to newest, by the
+// time when block is detached from the core.
+//
+// census_log_read_next() first iterates over dirty_block_list and then
+// core_local_blocks[]. It moves completely read blocks from dirty_block_list
+// to free_block_list. Blocks in core_local_blocks[] are not freed, even when
+// completely read.
+//
+// If the log is configured to discard old records and free_block_list is empty,
+// census_log_start_write() iterates over dirty_block_list to allocate a
+// new block. It moves the oldest available block (no pending read/write) to
+// core_local_blocks[].
+//
+// core_local_block_struct is used to implement a map from core id to the block
+// associated with that core. This mapping is advisory. It is possible that the
+// block returned by this mapping is no longer associated with that core. This
+// mapping is updated, lazily, by census_log_start_write().
+//
+// Locking in block struct:
+//
+// Exclusive g_log.lock must be held before calling any functions operating on
+// block structs except census_log_start_write() and census_log_end_write().
+//
+// Writes to a block are serialized via writer_lock. census_log_start_write()
+// acquires this lock and census_log_end_write() releases it. On failure to
+// acquire the lock, writer allocates a new block for the current core and
+// updates core_local_block accordingly.
+//
+// Simultaneous read and write access is allowed. Readers can safely read up to
+// committed bytes (bytes_committed).
+//
+// reader_lock protects the block, currently being read, from getting recycled.
+// start_read() acquires reader_lock and end_read() releases the lock.
+//
+// Read/write access to a block is disabled via try_disable_access(). It returns
+// with both writer_lock and reader_lock held. These locks are subsequently
+// released by enable_access() to enable access to the block.
+//
+// A note on naming: Most function/struct names are prepended by cl_
+// (shorthand for census_log). Further, functions that manipulate structures
+// include the name of the structure, which will be passed as the first
+// argument. E.g. cl_block_initialize() will initialize a cl_block.
+
+#include "src/core/census/log.h"
+#include <grpc/support/alloc.h>
+#include <grpc/support/atm.h>
+#include <grpc/support/cpu.h>
+#include <grpc/support/log.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/useful.h>
+#include <stdbool.h>
+#include <string.h>
+
+// End of platform specific code
+
+typedef struct census_log_block_list_struct {
+  struct census_log_block_list_struct* next;
+  struct census_log_block_list_struct* prev;
+  struct census_log_block* block;
+} cl_block_list_struct;
+
+typedef struct census_log_block {
+  // Pointer to underlying buffer.
+  char* buffer;
+  gpr_atm writer_lock;
+  gpr_atm reader_lock;
+  // Keeps completely written bytes. Declared atomic because accessed
+  // simultaneously by reader and writer.
+  gpr_atm bytes_committed;
+  // Bytes already read.
+  size_t bytes_read;
+  // Links for list.
+  cl_block_list_struct link;
+// We want this structure to be cacheline aligned. We assume the following
+// sizes for the various parts on 32/64bit systems:
+// type                 32b size    64b size
+// char*                   4           8
+// 3x gpr_atm             12          24
+// size_t                  4           8
+// cl_block_list_struct   12          24
+// TOTAL                  32          64
+//
+// Depending on the size of our cacheline and the architecture, we
+// selectively add char buffering to this structure. The size is checked
+// via assert in census_log_initialize().
+#if defined(GPR_ARCH_64)
+#define CL_BLOCK_PAD_SIZE (GPR_CACHELINE_SIZE - 64)
+#else
+#if defined(GPR_ARCH_32)
+#define CL_BLOCK_PAD_SIZE (GPR_CACHELINE_SIZE - 32)
+#else
+#error "Unknown architecture"
+#endif
+#endif
+#if CL_BLOCK_PAD_SIZE > 0
+  char padding[CL_BLOCK_PAD_SIZE];
+#endif
+} cl_block;
+
+// A list of cl_blocks, doubly-linked through cl_block::link.
+typedef struct census_log_block_list {
+  int32_t count;            // Number of items in list.
+  cl_block_list_struct ht;  // head/tail of linked list.
+} cl_block_list;
+
+// Cacheline aligned block pointers to avoid false sharing. Block pointer must
+// be initialized via set_block(), before calling other functions
+typedef struct census_log_core_local_block {
+  gpr_atm block;
+// Ensure cachline alignment: we assume sizeof(gpr_atm) == 4 or 8
+#if defined(GPR_ARCH_64)
+#define CL_CORE_LOCAL_BLOCK_PAD_SIZE (GPR_CACHELINE_SIZE - 8)
+#else
+#if defined(GPR_ARCH_32)
+#define CL_CORE_LOCAL_BLOCK_PAD_SIZE (GPR_CACHELINE_SIZE - 4)
+#else
+#error "Unknown architecture"
+#endif
+#endif
+#if CL_CORE_LOCAL_BLOCK_PAD_SIZE > 0
+  char padding[CL_CORE_LOCAL_BLOCK_PAD_SIZE];
+#endif
+} cl_core_local_block;
+
+struct census_log {
+  int discard_old_records;
+  // Number of cores (aka hardware-contexts)
+  unsigned num_cores;
+  // number of CENSUS_LOG_2_MAX_RECORD_SIZE blocks in log
+  uint32_t num_blocks;
+  cl_block* blocks;                        // Block metadata.
+  cl_core_local_block* core_local_blocks;  // Keeps core to block mappings.
+  gpr_mu lock;
+  int initialized;  // has log been initialized?
+  // Keeps the state of the reader iterator. A value of 0 indicates that
+  // iterator has reached the end. census_log_init_reader() resets the value
+  // to num_core to restart iteration.
+  uint32_t read_iterator_state;
+  // Points to the block being read. If non-NULL, the block is locked for
+  // reading(block_being_read_->reader_lock is held).
+  cl_block* block_being_read;
+  char* buffer;
+  cl_block_list free_block_list;
+  cl_block_list dirty_block_list;
+  gpr_atm out_of_space_count;
+};
+
+// Single internal log.
+static struct census_log g_log;
+
+// Functions that operate on an atomic memory location used as a lock.
+
+// Returns non-zero if lock is acquired.
+static int cl_try_lock(gpr_atm* lock) { return gpr_atm_acq_cas(lock, 0, 1); }
+
+static void cl_unlock(gpr_atm* lock) { gpr_atm_rel_store(lock, 0); }
+
+// Functions that operate on cl_core_local_block's.
+
+static void cl_core_local_block_set_block(cl_core_local_block* clb,
+                                          cl_block* block) {
+  gpr_atm_rel_store(&clb->block, (gpr_atm)block);
+}
+
+static cl_block* cl_core_local_block_get_block(cl_core_local_block* clb) {
+  return (cl_block*)gpr_atm_acq_load(&clb->block);
+}
+
+// Functions that operate on cl_block_list_struct's.
+
+static void cl_block_list_struct_initialize(cl_block_list_struct* bls,
+                                            cl_block* block) {
+  bls->next = bls->prev = bls;
+  bls->block = block;
+}
+
+// Functions that operate on cl_block_list's.
+
+static void cl_block_list_initialize(cl_block_list* list) {
+  list->count = 0;
+  cl_block_list_struct_initialize(&list->ht, NULL);
+}
+
+// Returns head of *this, or NULL if empty.
+static cl_block* cl_block_list_head(cl_block_list* list) {
+  return list->ht.next->block;
+}
+
+// Insert element *e after *pos.
+static void cl_block_list_insert(cl_block_list* list, cl_block_list_struct* pos,
+                                 cl_block_list_struct* e) {
+  list->count++;
+  e->next = pos->next;
+  e->prev = pos;
+  e->next->prev = e;
+  e->prev->next = e;
+}
+
+// Insert block at the head of the list
+static void cl_block_list_insert_at_head(cl_block_list* list, cl_block* block) {
+  cl_block_list_insert(list, &list->ht, &block->link);
+}
+
+// Insert block at the tail of the list.
+static void cl_block_list_insert_at_tail(cl_block_list* list, cl_block* block) {
+  cl_block_list_insert(list, list->ht.prev, &block->link);
+}
+
+// Removes block *b. Requires *b be in the list.
+static void cl_block_list_remove(cl_block_list* list, cl_block* b) {
+  list->count--;
+  b->link.next->prev = b->link.prev;
+  b->link.prev->next = b->link.next;
+}
+
+// Functions that operate on cl_block's
+
+static void cl_block_initialize(cl_block* block, char* buffer) {
+  block->buffer = buffer;
+  gpr_atm_rel_store(&block->writer_lock, 0);
+  gpr_atm_rel_store(&block->reader_lock, 0);
+  gpr_atm_rel_store(&block->bytes_committed, 0);
+  block->bytes_read = 0;
+  cl_block_list_struct_initialize(&block->link, block);
+}
+
+// Guards against exposing partially written buffer to the reader.
+static void cl_block_set_bytes_committed(cl_block* block,
+                                         size_t bytes_committed) {
+  gpr_atm_rel_store(&block->bytes_committed, (gpr_atm)bytes_committed);
+}
+
+static size_t cl_block_get_bytes_committed(cl_block* block) {
+  return (size_t)gpr_atm_acq_load(&block->bytes_committed);
+}
+
+// Tries to disable future read/write access to this block. Succeeds if:
+// - no in-progress write AND
+// - no in-progress read AND
+// - 'discard_data' set to true OR no unread data
+// On success, clears the block state and returns with writer_lock_ and
+// reader_lock_ held. These locks are released by a subsequent
+// cl_block_access_enable() call.
+static bool cl_block_try_disable_access(cl_block* block, int discard_data) {
+  if (!cl_try_lock(&block->writer_lock)) {
+    return false;
+  }
+  if (!cl_try_lock(&block->reader_lock)) {
+    cl_unlock(&block->writer_lock);
+    return false;
+  }
+  if (!discard_data &&
+      (block->bytes_read != cl_block_get_bytes_committed(block))) {
+    cl_unlock(&block->reader_lock);
+    cl_unlock(&block->writer_lock);
+    return false;
+  }
+  cl_block_set_bytes_committed(block, 0);
+  block->bytes_read = 0;
+  return true;
+}
+
+static void cl_block_enable_access(cl_block* block) {
+  cl_unlock(&block->reader_lock);
+  cl_unlock(&block->writer_lock);
+}
+
+// Returns with writer_lock held.
+static void* cl_block_start_write(cl_block* block, size_t size) {
+  if (!cl_try_lock(&block->writer_lock)) {
+    return NULL;
+  }
+  size_t bytes_committed = cl_block_get_bytes_committed(block);
+  if (bytes_committed + size > CENSUS_LOG_MAX_RECORD_SIZE) {
+    cl_unlock(&block->writer_lock);
+    return NULL;
+  }
+  return block->buffer + bytes_committed;
+}
+
+// Releases writer_lock and increments committed bytes by 'bytes_written'.
+// 'bytes_written' must be <= 'size' specified in the corresponding
+// StartWrite() call. This function is thread-safe.
+static void cl_block_end_write(cl_block* block, size_t bytes_written) {
+  cl_block_set_bytes_committed(
+      block, cl_block_get_bytes_committed(block) + bytes_written);
+  cl_unlock(&block->writer_lock);
+}
+
+// Returns a pointer to the first unread byte in buffer. The number of bytes
+// available are returned in 'bytes_available'. Acquires reader lock that is
+// released by a subsequent cl_block_end_read() call. Returns NULL if:
+// - read in progress
+// - no data available
+static void* cl_block_start_read(cl_block* block, size_t* bytes_available) {
+  if (!cl_try_lock(&block->reader_lock)) {
+    return NULL;
+  }
+  // bytes_committed may change from under us. Use bytes_available to update
+  // bytes_read below.
+  size_t bytes_committed = cl_block_get_bytes_committed(block);
+  GPR_ASSERT(bytes_committed >= block->bytes_read);
+  *bytes_available = bytes_committed - block->bytes_read;
+  if (*bytes_available == 0) {
+    cl_unlock(&block->reader_lock);
+    return NULL;
+  }
+  void* record = block->buffer + block->bytes_read;
+  block->bytes_read += *bytes_available;
+  return record;
+}
+
+static void cl_block_end_read(cl_block* block) {
+  cl_unlock(&block->reader_lock);
+}
+
+// Internal functions operating on g_log
+
+// Allocates a new free block (or recycles an available dirty block if log is
+// configured to discard old records). Returns NULL if out-of-space.
+static cl_block* cl_allocate_block(void) {
+  cl_block* block = cl_block_list_head(&g_log.free_block_list);
+  if (block != NULL) {
+    cl_block_list_remove(&g_log.free_block_list, block);
+    return block;
+  }
+  if (!g_log.discard_old_records) {
+    // No free block and log is configured to keep old records.
+    return NULL;
+  }
+  // Recycle dirty block. Start from the oldest.
+  for (block = cl_block_list_head(&g_log.dirty_block_list); block != NULL;
+       block = block->link.next->block) {
+    if (cl_block_try_disable_access(block, 1 /* discard data */)) {
+      cl_block_list_remove(&g_log.dirty_block_list, block);
+      return block;
+    }
+  }
+  return NULL;
+}
+
+// Allocates a new block and updates core id => block mapping. 'old_block'
+// points to the block that the caller thinks is attached to
+// 'core_id'. 'old_block' may be NULL. Returns true if:
+// - allocated a new block OR
+// - 'core_id' => 'old_block' mapping changed (another thread allocated a
+//   block before lock was acquired).
+static bool cl_allocate_core_local_block(uint32_t core_id,
+                                         cl_block* old_block) {
+  // Now that we have the lock, check if core-local mapping has changed.
+  cl_core_local_block* core_local_block = &g_log.core_local_blocks[core_id];
+  cl_block* block = cl_core_local_block_get_block(core_local_block);
+  if ((block != NULL) && (block != old_block)) {
+    return true;
+  }
+  if (block != NULL) {
+    cl_core_local_block_set_block(core_local_block, NULL);
+    cl_block_list_insert_at_tail(&g_log.dirty_block_list, block);
+  }
+  block = cl_allocate_block();
+  if (block == NULL) {
+    return false;
+  }
+  cl_core_local_block_set_block(core_local_block, block);
+  cl_block_enable_access(block);
+  return true;
+}
+
+static cl_block* cl_get_block(void* record) {
+  uintptr_t p = (uintptr_t)((char*)record - g_log.buffer);
+  uintptr_t index = p >> CENSUS_LOG_2_MAX_RECORD_SIZE;
+  return &g_log.blocks[index];
+}
+
+// Gets the next block to read and tries to free 'prev' block (if not NULL).
+// Returns NULL if reached the end.
+static cl_block* cl_next_block_to_read(cl_block* prev) {
+  cl_block* block = NULL;
+  if (g_log.read_iterator_state == g_log.num_cores) {
+    // We are traversing dirty list; find the next dirty block.
+    if (prev != NULL) {
+      // Try to free the previous block if there is no unread data. This
+      // block
+      // may have unread data if previously incomplete record completed
+      // between
+      // read_next() calls.
+      block = prev->link.next->block;
+      if (cl_block_try_disable_access(prev, 0 /* do not discard data */)) {
+        cl_block_list_remove(&g_log.dirty_block_list, prev);
+        cl_block_list_insert_at_head(&g_log.free_block_list, prev);
+      }
+    } else {
+      block = cl_block_list_head(&g_log.dirty_block_list);
+    }
+    if (block != NULL) {
+      return block;
+    }
+    // We are done with the dirty list; moving on to core-local blocks.
+  }
+  while (g_log.read_iterator_state > 0) {
+    g_log.read_iterator_state--;
+    block = cl_core_local_block_get_block(
+        &g_log.core_local_blocks[g_log.read_iterator_state]);
+    if (block != NULL) {
+      return block;
+    }
+  }
+  return NULL;
+}
+
+#define CL_LOG_2_MB 20  // 2^20 = 1MB
+
+// External functions: primary stats_log interface
+void census_log_initialize(size_t size_in_mb, int discard_old_records) {
+  // Check cacheline alignment.
+  GPR_ASSERT(sizeof(cl_block) % GPR_CACHELINE_SIZE == 0);
+  GPR_ASSERT(sizeof(cl_core_local_block) % GPR_CACHELINE_SIZE == 0);
+  GPR_ASSERT(!g_log.initialized);
+  g_log.discard_old_records = discard_old_records;
+  g_log.num_cores = gpr_cpu_num_cores();
+  // Ensure that we will not get any overflow in calaculating num_blocks
+  GPR_ASSERT(CL_LOG_2_MB >= CENSUS_LOG_2_MAX_RECORD_SIZE);
+  GPR_ASSERT(size_in_mb < 1000);
+  // Ensure at least 2x as many blocks as there are cores.
+  g_log.num_blocks =
+      (uint32_t)GPR_MAX(2 * g_log.num_cores, (size_in_mb << CL_LOG_2_MB) >>
+                                                 CENSUS_LOG_2_MAX_RECORD_SIZE);
+  gpr_mu_init(&g_log.lock);
+  g_log.read_iterator_state = 0;
+  g_log.block_being_read = NULL;
+  g_log.core_local_blocks = (cl_core_local_block*)gpr_malloc_aligned(
+      g_log.num_cores * sizeof(cl_core_local_block), GPR_CACHELINE_SIZE_LOG);
+  memset(g_log.core_local_blocks, 0,
+         g_log.num_cores * sizeof(cl_core_local_block));
+  g_log.blocks = (cl_block*)gpr_malloc_aligned(
+      g_log.num_blocks * sizeof(cl_block), GPR_CACHELINE_SIZE_LOG);
+  memset(g_log.blocks, 0, g_log.num_blocks * sizeof(cl_block));
+  g_log.buffer = gpr_malloc(g_log.num_blocks * CENSUS_LOG_MAX_RECORD_SIZE);
+  memset(g_log.buffer, 0, g_log.num_blocks * CENSUS_LOG_MAX_RECORD_SIZE);
+  cl_block_list_initialize(&g_log.free_block_list);
+  cl_block_list_initialize(&g_log.dirty_block_list);
+  for (uint32_t i = 0; i < g_log.num_blocks; ++i) {
+    cl_block* block = g_log.blocks + i;
+    cl_block_initialize(block, g_log.buffer + (CENSUS_LOG_MAX_RECORD_SIZE * i));
+    cl_block_try_disable_access(block, 1 /* discard data */);
+    cl_block_list_insert_at_tail(&g_log.free_block_list, block);
+  }
+  gpr_atm_rel_store(&g_log.out_of_space_count, 0);
+  g_log.initialized = 1;
+}
+
+void census_log_shutdown(void) {
+  GPR_ASSERT(g_log.initialized);
+  gpr_mu_destroy(&g_log.lock);
+  gpr_free_aligned(g_log.core_local_blocks);
+  g_log.core_local_blocks = NULL;
+  gpr_free_aligned(g_log.blocks);
+  g_log.blocks = NULL;
+  gpr_free(g_log.buffer);
+  g_log.buffer = NULL;
+  g_log.initialized = 0;
+}
+
+void* census_log_start_write(size_t size) {
+  // Used to bound number of times block allocation is attempted.
+  GPR_ASSERT(size > 0);
+  GPR_ASSERT(g_log.initialized);
+  if (size > CENSUS_LOG_MAX_RECORD_SIZE) {
+    return NULL;
+  }
+  uint32_t attempts_remaining = g_log.num_blocks;
+  uint32_t core_id = gpr_cpu_current_cpu();
+  do {
+    void* record = NULL;
+    cl_block* block =
+        cl_core_local_block_get_block(&g_log.core_local_blocks[core_id]);
+    if (block && (record = cl_block_start_write(block, size))) {
+      return record;
+    }
+    // Need to allocate a new block. We are here if:
+    // - No block associated with the core OR
+    // - Write in-progress on the block OR
+    // - block is out of space
+    gpr_mu_lock(&g_log.lock);
+    bool allocated = cl_allocate_core_local_block(core_id, block);
+    gpr_mu_unlock(&g_log.lock);
+    if (!allocated) {
+      gpr_atm_no_barrier_fetch_add(&g_log.out_of_space_count, 1);
+      return NULL;
+    }
+  } while (attempts_remaining--);
+  // Give up.
+  gpr_atm_no_barrier_fetch_add(&g_log.out_of_space_count, 1);
+  return NULL;
+}
+
+void census_log_end_write(void* record, size_t bytes_written) {
+  GPR_ASSERT(g_log.initialized);
+  cl_block_end_write(cl_get_block(record), bytes_written);
+}
+
+void census_log_init_reader(void) {
+  GPR_ASSERT(g_log.initialized);
+  gpr_mu_lock(&g_log.lock);
+  // If a block is locked for reading unlock it.
+  if (g_log.block_being_read != NULL) {
+    cl_block_end_read(g_log.block_being_read);
+    g_log.block_being_read = NULL;
+  }
+  g_log.read_iterator_state = g_log.num_cores;
+  gpr_mu_unlock(&g_log.lock);
+}
+
+const void* census_log_read_next(size_t* bytes_available) {
+  GPR_ASSERT(g_log.initialized);
+  gpr_mu_lock(&g_log.lock);
+  if (g_log.block_being_read != NULL) {
+    cl_block_end_read(g_log.block_being_read);
+  }
+  do {
+    g_log.block_being_read = cl_next_block_to_read(g_log.block_being_read);
+    if (g_log.block_being_read != NULL) {
+      void* record =
+          cl_block_start_read(g_log.block_being_read, bytes_available);
+      if (record != NULL) {
+        gpr_mu_unlock(&g_log.lock);
+        return record;
+      }
+    }
+  } while (g_log.block_being_read != NULL);
+  gpr_mu_unlock(&g_log.lock);
+  return NULL;
+}
+
+size_t census_log_remaining_space(void) {
+  GPR_ASSERT(g_log.initialized);
+  size_t space = 0;
+  gpr_mu_lock(&g_log.lock);
+  if (g_log.discard_old_records) {
+    // Remaining space is not meaningful; just return the entire log space.
+    space = g_log.num_blocks << CENSUS_LOG_2_MAX_RECORD_SIZE;
+  } else {
+    GPR_ASSERT(g_log.free_block_list.count >= 0);
+    space = (size_t)g_log.free_block_list.count * CENSUS_LOG_MAX_RECORD_SIZE;
+  }
+  gpr_mu_unlock(&g_log.lock);
+  return space;
+}
+
+int64_t census_log_out_of_space_count(void) {
+  GPR_ASSERT(g_log.initialized);
+  return gpr_atm_acq_load(&g_log.out_of_space_count);
+}
diff --git a/src/core/census/log.h b/src/core/census/log.h
new file mode 100644
index 0000000000..05daea066f
--- /dev/null
+++ b/src/core/census/log.h
@@ -0,0 +1,93 @@
+/*
+ *
+ * Copyright 2015-2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_INTERNAL_CORE_CENSUS_LOG_H
+#define GRPC_INTERNAL_CORE_CENSUS_LOG_H
+
+#include <grpc/support/port_platform.h>
+#include <stddef.h>
+
+/* Maximum record size, in bytes. */
+#define CENSUS_LOG_2_MAX_RECORD_SIZE 14 /* 2^14 = 16KB */
+#define CENSUS_LOG_MAX_RECORD_SIZE (1 << CENSUS_LOG_2_MAX_RECORD_SIZE)
+
+/* Initialize the statistics logging subsystem with the given log size. A log
+   size of 0 will result in the smallest possible log for the platform
+   (approximately CENSUS_LOG_MAX_RECORD_SIZE * gpr_cpu_num_cores()). If
+   discard_old_records is non-zero, then new records will displace older ones
+   when the log is full. This function must be called before any other
+   census_log functions.
+*/
+void census_log_initialize(size_t size_in_mb, int discard_old_records);
+
+/* Shutdown the logging subsystem. Caller must ensure that:
+   - no in progress or future call to any census_log functions
+   - no incomplete records
+*/
+void census_log_shutdown(void);
+
+/* Allocates and returns a 'size' bytes record and marks it in use. A
+   subsequent census_log_end_write() marks the record complete. The
+   'bytes_written' census_log_end_write() argument must be <=
+   'size'. Returns NULL if out-of-space AND:
+       - log is configured to keep old records OR
+       - all blocks are pinned by incomplete records.
+*/
+void* census_log_start_write(size_t size);
+
+void census_log_end_write(void* record, size_t bytes_written);
+
+void census_log_init_reader(void);
+
+/* census_log_read_next() iterates over blocks with data and for each block
+   returns a pointer to the first unread byte. The number of bytes that can be
+   read are returned in 'bytes_available'. Reader is expected to read all
+   available data. Reading the data consumes it i.e. it cannot be read again.
+   census_log_read_next() returns NULL if the end is reached i.e last block
+   is read. census_log_init_reader() starts the iteration or aborts the
+   current iteration.
+*/
+const void* census_log_read_next(size_t* bytes_available);
+
+/* Returns estimated remaining space across all blocks, in bytes. If log is
+   configured to discard old records, returns total log space. Otherwise,
+   returns space available in empty blocks (partially filled blocks are
+   treated as full).
+*/
+size_t census_log_remaining_space(void);
+
+/* Returns the number of times gprc_stats_log_start_write() failed due to
+   out-of-space. */
+int64_t census_log_out_of_space_count(void);
+
+#endif /* GRPC_INTERNAL_CORE_CENSUS_LOG_H */
diff --git a/src/core/channel/http_client_filter.c b/src/core/channel/http_client_filter.c
index 43eee046b8..1aa27208c2 100644
--- a/src/core/channel/http_client_filter.c
+++ b/src/core/channel/http_client_filter.c
@@ -127,8 +127,8 @@ static void hc_mutate_op(grpc_call_element *elem,
   if (op->recv_initial_metadata != NULL) {
     /* substitute our callback for the higher callback */
     calld->recv_initial_metadata = op->recv_initial_metadata;
-    calld->on_done_recv = op->on_complete;
-    op->on_complete = &calld->hc_on_recv;
+    calld->on_done_recv = op->recv_initial_metadata_ready;
+    op->recv_initial_metadata_ready = &calld->hc_on_recv;
   }
 }
 
diff --git a/src/core/channel/http_server_filter.c b/src/core/channel/http_server_filter.c
index bb75323933..370f8dbe42 100644
--- a/src/core/channel/http_server_filter.c
+++ b/src/core/channel/http_server_filter.c
@@ -186,8 +186,8 @@ static void hs_mutate_op(grpc_call_element *elem,
   if (op->recv_initial_metadata) {
     /* substitute our callback for the higher callback */
     calld->recv_initial_metadata = op->recv_initial_metadata;
-    calld->on_done_recv = op->on_complete;
-    op->on_complete = &calld->hs_on_recv;
+    calld->on_done_recv = op->recv_initial_metadata_ready;
+    op->recv_initial_metadata_ready = &calld->hs_on_recv;
   }
 }
 
diff --git a/src/core/channel/subchannel_call_holder.c b/src/core/channel/subchannel_call_holder.c
index 3ad9fd9efb..81297c8d44 100644
--- a/src/core/channel/subchannel_call_holder.c
+++ b/src/core/channel/subchannel_call_holder.c
@@ -241,10 +241,8 @@ static void fail_locked(grpc_exec_ctx *exec_ctx,
                         grpc_subchannel_call_holder *holder) {
   size_t i;
   for (i = 0; i < holder->waiting_ops_count; i++) {
-    grpc_exec_ctx_enqueue(exec_ctx, holder->waiting_ops[i].on_complete, false,
-                          NULL);
-    grpc_exec_ctx_enqueue(exec_ctx, holder->waiting_ops[i].recv_message_ready,
-                          false, NULL);
+    grpc_transport_stream_op_finish_with_failure(exec_ctx,
+                                                 &holder->waiting_ops[i]);
   }
   holder->waiting_ops_count = 0;
 }
diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c
index 2e650ca939..02c6678363 100644
--- a/src/core/iomgr/pollset_windows.c
+++ b/src/core/iomgr/pollset_windows.c
@@ -212,10 +212,8 @@ void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) {
         grpc_iocp_kick();
       }
     } else {
-      if (p->is_iocp_worker) {
-        if (g_active_poller == specific_worker) {
-          grpc_iocp_kick();
-        }
+      if (p->is_iocp_worker && g_active_poller == specific_worker) {
+        grpc_iocp_kick();
       } else {
         specific_worker->kicked = 1;
         gpr_cv_signal(&specific_worker->cv);
diff --git a/src/core/iomgr/udp_server.c b/src/core/iomgr/udp_server.c
index fe006c603c..ef548cfe4d 100644
--- a/src/core/iomgr/udp_server.c
+++ b/src/core/iomgr/udp_server.c
@@ -137,7 +137,7 @@ grpc_udp_server *grpc_udp_server_create(void) {
 }
 
 static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) {
-  grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, 1);
+  grpc_exec_ctx_enqueue(exec_ctx, s->shutdown_complete, 1, NULL);
 
   gpr_mu_destroy(&s->mu);
   gpr_cv_destroy(&s->cv);
@@ -146,7 +146,8 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) {
   gpr_free(s);
 }
 
-static void destroyed_port(grpc_exec_ctx *exec_ctx, void *server, int success) {
+static void destroyed_port(grpc_exec_ctx *exec_ctx, void *server,
+                           bool success) {
   grpc_udp_server *s = server;
   gpr_mu_lock(&s->mu);
   s->destroyed_ports++;
@@ -263,10 +264,10 @@ error:
 }
 
 /* event manager callback when reads are ready */
-static void on_read(grpc_exec_ctx *exec_ctx, void *arg, int success) {
+static void on_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   server_port *sp = arg;
 
-  if (success == 0) {
+  if (!success) {
     gpr_mu_lock(&sp->server->mu);
     if (0 == --sp->server->active_ports) {
       gpr_mu_unlock(&sp->server->mu);
diff --git a/src/core/security/server_auth_filter.c b/src/core/security/server_auth_filter.c
index 4c78711387..3d8e5e8d35 100644
--- a/src/core/security/server_auth_filter.c
+++ b/src/core/security/server_auth_filter.c
@@ -176,8 +176,8 @@ static void set_recv_ops_md_callbacks(grpc_call_element *elem,
   if (op->recv_initial_metadata != NULL) {
     /* substitute our callback for the higher callback */
     calld->recv_initial_metadata = op->recv_initial_metadata;
-    calld->on_done_recv = op->on_complete;
-    op->on_complete = &calld->auth_on_recv;
+    calld->on_done_recv = op->recv_initial_metadata_ready;
+    op->recv_initial_metadata_ready = &calld->auth_on_recv;
     calld->transport_op = *op;
   }
 }
diff --git a/src/core/support/env_linux.c b/src/core/support/env_linux.c
index 442cd8298e..1ca6fa1aff 100644
--- a/src/core/support/env_linux.c
+++ b/src/core/support/env_linux.c
@@ -52,6 +52,7 @@
 #include "src/core/support/string.h"
 
 char *gpr_getenv(const char *name) {
+#if defined(GPR_BACKWARDS_COMPATIBILITY_MODE)
   typedef char *(*getenv_type)(const char *);
   static getenv_type getenv_func = NULL;
   /* Check to see which getenv variant is supported (go from most
@@ -62,6 +63,10 @@ char *gpr_getenv(const char *name) {
   }
   char *result = getenv_func(name);
   return result == NULL ? result : gpr_strdup(result);
+#else
+  char *result = secure_getenv(name);
+  return result == NULL ? result : gpr_strdup(result);
+#endif
 }
 
 void gpr_setenv(const char *name, const char *value) {
diff --git a/src/core/support/time_posix.c b/src/core/support/time_posix.c
index 1f92d7f090..36d75e7da2 100644
--- a/src/core/support/time_posix.c
+++ b/src/core/support/time_posix.c
@@ -86,7 +86,7 @@ gpr_timespec gpr_now(gpr_clock_type clock_type) {
     gpr_precise_clock_now(&ret);
     return ret;
   } else {
-#if defined(__linux__) && !defined(GPR_NO_DIRECT_SYSCALLS)
+#if defined(GPR_BACKWARDS_COMPATIBILITY_MODE) && defined(__linux__)
     /* avoid ABI problems by invoking syscalls directly */
     syscall(SYS_clock_gettime, clockid_for_gpr_clock[clock_type], &now);
 #else
diff --git a/src/core/surface/alarm.c b/src/core/surface/alarm.c
index d753023ca9..fb496f6c47 100644
--- a/src/core/surface/alarm.c
+++ b/src/core/surface/alarm.c
@@ -63,9 +63,9 @@ grpc_alarm *grpc_alarm_create(grpc_completion_queue *cq, gpr_timespec deadline,
   alarm->cq = cq;
   alarm->tag = tag;
 
+  grpc_cq_begin_op(cq, tag);
   grpc_timer_init(&exec_ctx, &alarm->alarm, deadline, alarm_cb, alarm,
                   gpr_now(GPR_CLOCK_MONOTONIC));
-  grpc_cq_begin_op(cq, tag);
   grpc_exec_ctx_finish(&exec_ctx);
   return alarm;
 }
diff --git a/src/core/surface/call.c b/src/core/surface/call.c
index 9495e748b5..1b117aa6b8 100644
--- a/src/core/surface/call.c
+++ b/src/core/surface/call.c
@@ -159,6 +159,9 @@ struct grpc_call {
   uint8_t receiving_message;
   uint8_t received_final_op;
 
+  /* have we received initial metadata */
+  bool has_initial_md_been_received;
+
   batch_control active_batches[MAX_CONCURRENT_BATCHES];
 
   /* first idx: is_receiving, second idx: is_trailing */
@@ -200,6 +203,7 @@ struct grpc_call {
   gpr_slice receiving_slice;
   grpc_closure receiving_slice_ready;
   grpc_closure receiving_stream_ready;
+  grpc_closure receiving_initial_metadata_ready;
   uint32_t test_only_last_message_flags;
 
   union {
@@ -212,6 +216,11 @@ struct grpc_call {
       int *cancelled;
     } server;
   } final_op;
+
+  struct {
+    void *bctlp;
+    bool success;
+  } saved_receiving_stream_ready_ctx;
 };
 
 #define CALL_STACK_FROM_CALL(call) ((grpc_call_stack *)((call) + 1))
@@ -993,6 +1002,94 @@ static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
   }
 }
 
+static void process_data_after_md(grpc_exec_ctx *exec_ctx, batch_control *bctl,
+                                  bool success) {
+  grpc_call *call = bctl->call;
+  if (call->receiving_stream == NULL) {
+    *call->receiving_buffer = NULL;
+    call->receiving_message = 0;
+    if (gpr_unref(&bctl->steps_to_complete)) {
+      post_batch_completion(exec_ctx, bctl);
+    }
+  } else if (call->receiving_stream->length >
+             grpc_channel_get_max_message_length(call->channel)) {
+    cancel_with_status(exec_ctx, call, GRPC_STATUS_INTERNAL,
+                       "Max message size exceeded");
+    grpc_byte_stream_destroy(exec_ctx, call->receiving_stream);
+    call->receiving_stream = NULL;
+    *call->receiving_buffer = NULL;
+    call->receiving_message = 0;
+    if (gpr_unref(&bctl->steps_to_complete)) {
+      post_batch_completion(exec_ctx, bctl);
+    }
+  } else {
+    call->test_only_last_message_flags = call->receiving_stream->flags;
+    if ((call->receiving_stream->flags & GRPC_WRITE_INTERNAL_COMPRESS) &&
+        (call->compression_algorithm > GRPC_COMPRESS_NONE)) {
+      *call->receiving_buffer = grpc_raw_compressed_byte_buffer_create(
+          NULL, 0, call->compression_algorithm);
+    } else {
+      *call->receiving_buffer = grpc_raw_byte_buffer_create(NULL, 0);
+    }
+    grpc_closure_init(&call->receiving_slice_ready, receiving_slice_ready,
+                      bctl);
+    continue_receiving_slices(exec_ctx, bctl);
+    /* early out */
+    return;
+  }
+}
+
+static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
+                                   bool success) {
+  batch_control *bctl = bctlp;
+  grpc_call *call = bctl->call;
+
+  gpr_mu_lock(&bctl->call->mu);
+  if (bctl->call->has_initial_md_been_received) {
+    gpr_mu_unlock(&bctl->call->mu);
+    process_data_after_md(exec_ctx, bctlp, success);
+  } else {
+    call->saved_receiving_stream_ready_ctx.bctlp = bctlp;
+    call->saved_receiving_stream_ready_ctx.success = success;
+    gpr_mu_unlock(&bctl->call->mu);
+  }
+}
+
+static void receiving_initial_metadata_ready(grpc_exec_ctx *exec_ctx,
+                                             void *bctlp, bool success) {
+  batch_control *bctl = bctlp;
+  grpc_call *call = bctl->call;
+
+  gpr_mu_lock(&call->mu);
+
+  grpc_metadata_batch *md =
+      &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */];
+  grpc_metadata_batch_filter(md, recv_initial_filter, call);
+  call->has_initial_md_been_received = true;
+
+  if (gpr_time_cmp(md->deadline, gpr_inf_future(md->deadline.clock_type)) !=
+          0 &&
+      !call->is_client) {
+    GPR_TIMER_BEGIN("set_deadline_alarm", 0);
+    set_deadline_alarm(exec_ctx, call, md->deadline);
+    GPR_TIMER_END("set_deadline_alarm", 0);
+  }
+
+  if (call->saved_receiving_stream_ready_ctx.bctlp != NULL) {
+    grpc_closure *saved_rsr_closure = grpc_closure_create(
+        receiving_stream_ready, call->saved_receiving_stream_ready_ctx.bctlp);
+    grpc_exec_ctx_enqueue(exec_ctx, saved_rsr_closure,
+                          call->saved_receiving_stream_ready_ctx.success, NULL);
+    call->saved_receiving_stream_ready_ctx.bctlp = NULL;
+  }
+
+  gpr_mu_unlock(&call->mu);
+
+  if (gpr_unref(&bctl->steps_to_complete)) {
+    post_batch_completion(exec_ctx, bctl);
+  }
+}
+
 static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp, bool success) {
   batch_control *bctl = bctlp;
   grpc_call *call = bctl->call;
@@ -1011,19 +1108,6 @@ static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp, bool success) {
     grpc_metadata_batch_destroy(
         &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */]);
   }
-  if (bctl->recv_initial_metadata) {
-    grpc_metadata_batch *md =
-        &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */];
-    grpc_metadata_batch_filter(md, recv_initial_filter, call);
-
-    if (gpr_time_cmp(md->deadline, gpr_inf_future(md->deadline.clock_type)) !=
-            0 &&
-        !call->is_client) {
-      GPR_TIMER_BEGIN("set_deadline_alarm", 0);
-      set_deadline_alarm(exec_ctx, call, md->deadline);
-      GPR_TIMER_END("set_deadline_alarm", 0);
-    }
-  }
   if (bctl->recv_final_op) {
     grpc_metadata_batch *md =
         &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */];
@@ -1065,45 +1149,6 @@ static void finish_batch(grpc_exec_ctx *exec_ctx, void *bctlp, bool success) {
   }
 }
 
-static void receiving_stream_ready(grpc_exec_ctx *exec_ctx, void *bctlp,
-                                   bool success) {
-  batch_control *bctl = bctlp;
-  grpc_call *call = bctl->call;
-
-  if (call->receiving_stream == NULL) {
-    *call->receiving_buffer = NULL;
-    call->receiving_message = 0;
-    if (gpr_unref(&bctl->steps_to_complete)) {
-      post_batch_completion(exec_ctx, bctl);
-    }
-  } else if (call->receiving_stream->length >
-             grpc_channel_get_max_message_length(call->channel)) {
-    cancel_with_status(exec_ctx, call, GRPC_STATUS_INTERNAL,
-                       "Max message size exceeded");
-    grpc_byte_stream_destroy(exec_ctx, call->receiving_stream);
-    call->receiving_stream = NULL;
-    *call->receiving_buffer = NULL;
-    call->receiving_message = 0;
-    if (gpr_unref(&bctl->steps_to_complete)) {
-      post_batch_completion(exec_ctx, bctl);
-    }
-  } else {
-    call->test_only_last_message_flags = call->receiving_stream->flags;
-    if ((call->receiving_stream->flags & GRPC_WRITE_INTERNAL_COMPRESS) &&
-        (call->compression_algorithm > GRPC_COMPRESS_NONE)) {
-      *call->receiving_buffer = grpc_raw_compressed_byte_buffer_create(
-          NULL, 0, call->compression_algorithm);
-    } else {
-      *call->receiving_buffer = grpc_raw_byte_buffer_create(NULL, 0);
-    }
-    grpc_closure_init(&call->receiving_slice_ready, receiving_slice_ready,
-                      bctl);
-    continue_receiving_slices(exec_ctx, bctl);
-    /* early out */
-    return;
-  }
-}
-
 static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
                                         grpc_call *call, const grpc_op *ops,
                                         size_t nops, void *notify_tag,
@@ -1273,9 +1318,14 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         }
         call->received_initial_metadata = 1;
         call->buffered_metadata[0] = op->data.recv_initial_metadata;
+        grpc_closure_init(&call->receiving_initial_metadata_ready,
+                          receiving_initial_metadata_ready, bctl);
         bctl->recv_initial_metadata = 1;
         stream_op.recv_initial_metadata =
             &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */];
+        stream_op.recv_initial_metadata_ready =
+            &call->receiving_initial_metadata_ready;
+        num_completion_callbacks_needed++;
         break;
       case GRPC_OP_RECV_MESSAGE:
         /* Flag validation: currently allow no flags */
diff --git a/src/core/surface/lame_client.c b/src/core/surface/lame_client.c
index 705996cad3..537069e984 100644
--- a/src/core/surface/lame_client.c
+++ b/src/core/surface/lame_client.c
@@ -78,8 +78,7 @@ static void lame_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
   } else if (op->recv_trailing_metadata != NULL) {
     fill_metadata(elem, op->recv_trailing_metadata);
   }
-  grpc_exec_ctx_enqueue(exec_ctx, op->on_complete, false, NULL);
-  grpc_exec_ctx_enqueue(exec_ctx, op->recv_message_ready, false, NULL);
+  grpc_transport_stream_op_finish_with_failure(exec_ctx, op);
 }
 
 static char *lame_get_peer(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) {
diff --git a/src/core/surface/server.c b/src/core/surface/server.c
index 42cffccb4c..fb5e0d4b9e 100644
--- a/src/core/surface/server.c
+++ b/src/core/surface/server.c
@@ -596,8 +596,8 @@ static void server_mutate_op(grpc_call_element *elem,
 
   if (op->recv_initial_metadata != NULL) {
     calld->recv_initial_metadata = op->recv_initial_metadata;
-    calld->on_done_recv_initial_metadata = op->on_complete;
-    op->on_complete = &calld->server_on_recv_initial_metadata;
+    calld->on_done_recv_initial_metadata = op->recv_initial_metadata_ready;
+    op->recv_initial_metadata_ready = &calld->server_on_recv_initial_metadata;
   }
 }
 
diff --git a/src/core/surface/validate_metadata.c b/src/core/surface/validate_metadata.c
index df2e80b4b7..bf4126867f 100644
--- a/src/core/surface/validate_metadata.c
+++ b/src/core/surface/validate_metadata.c
@@ -50,7 +50,7 @@ static int conforms_to(const char *s, size_t len, const uint8_t *legal_bits) {
 
 int grpc_header_key_is_legal(const char *key, size_t length) {
   static const uint8_t legal_header_bits[256 / 8] = {
-      0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xff, 0x03, 0x00, 0x00, 0x00,
+      0x00, 0x00, 0x00, 0x00, 0x00, 0x60, 0xff, 0x03, 0x00, 0x00, 0x00,
       0x80, 0xfe, 0xff, 0xff, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
   if (length == 0) {
diff --git a/src/core/transport/chttp2/internal.h b/src/core/transport/chttp2/internal.h
index c611496e7e..0e1e2c4265 100644
--- a/src/core/transport/chttp2/internal.h
+++ b/src/core/transport/chttp2/internal.h
@@ -385,7 +385,7 @@ typedef struct {
   grpc_closure *send_trailing_metadata_finished;
 
   grpc_metadata_batch *recv_initial_metadata;
-  grpc_closure *recv_initial_metadata_finished;
+  grpc_closure *recv_initial_metadata_ready;
   grpc_byte_stream **recv_message;
   grpc_closure *recv_message_ready;
   grpc_metadata_batch *recv_trailing_metadata;
diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c
index 9298573c7f..617d98875c 100644
--- a/src/core/transport/chttp2_transport.c
+++ b/src/core/transport/chttp2_transport.c
@@ -544,7 +544,7 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
   GPR_ASSERT(s->global.send_initial_metadata_finished == NULL);
   GPR_ASSERT(s->global.send_message_finished == NULL);
   GPR_ASSERT(s->global.send_trailing_metadata_finished == NULL);
-  GPR_ASSERT(s->global.recv_initial_metadata_finished == NULL);
+  GPR_ASSERT(s->global.recv_initial_metadata_ready == NULL);
   GPR_ASSERT(s->global.recv_message_ready == NULL);
   GPR_ASSERT(s->global.recv_trailing_metadata_finished == NULL);
   grpc_chttp2_data_parser_destroy(exec_ctx, &s->parsing.data_parser);
@@ -863,9 +863,9 @@ static void perform_stream_op_locked(
   }
 
   if (op->recv_initial_metadata != NULL) {
-    GPR_ASSERT(stream_global->recv_initial_metadata_finished == NULL);
-    stream_global->recv_initial_metadata_finished =
-        add_closure_barrier(on_complete);
+    GPR_ASSERT(stream_global->recv_initial_metadata_ready == NULL);
+    stream_global->recv_initial_metadata_ready =
+        op->recv_initial_metadata_ready;
     stream_global->recv_initial_metadata = op->recv_initial_metadata;
     grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
   }
@@ -1009,13 +1009,14 @@ static void check_read_ops(grpc_exec_ctx *exec_ctx,
   grpc_byte_stream *bs;
   while (
       grpc_chttp2_list_pop_check_read_ops(transport_global, &stream_global)) {
-    if (stream_global->recv_initial_metadata_finished != NULL &&
+    if (stream_global->recv_initial_metadata_ready != NULL &&
         stream_global->published_initial_metadata) {
       grpc_chttp2_incoming_metadata_buffer_publish(
           &stream_global->received_initial_metadata,
           stream_global->recv_initial_metadata);
-      grpc_chttp2_complete_closure_step(
-          exec_ctx, &stream_global->recv_initial_metadata_finished, 1);
+      grpc_exec_ctx_enqueue(
+          exec_ctx, stream_global->recv_initial_metadata_ready, true, NULL);
+      stream_global->recv_initial_metadata_ready = NULL;
     }
     if (stream_global->recv_message_ready != NULL) {
       if (stream_global->incoming_frames.head != NULL) {
diff --git a/src/core/transport/static_metadata.c b/src/core/transport/static_metadata.c
index 233e5c0d92..eeedae0619 100644
--- a/src/core/transport/static_metadata.c
+++ b/src/core/transport/static_metadata.c
@@ -35,11 +35,11 @@
  * WARNING: Auto-generated code.
  *
  * To make changes to this file, change
- *tools/codegen/core/gen_static_metadata.py,
+ * tools/codegen/core/gen_static_metadata.py,
  * and then re-run it.
  *
  * See metadata.h for an explanation of the interface here, and metadata.c for
- *an
+ * an
  * explanation of what's going on.
  */
 
@@ -69,21 +69,21 @@ const char *const grpc_static_metadata_strings[GRPC_STATIC_MDSTR_COUNT] = {
     "0", "1", "2", "200", "204", "206", "304", "400", "404", "500", "accept",
     "accept-charset", "accept-encoding", "accept-language", "accept-ranges",
     "access-control-allow-origin", "age", "allow", "application/grpc",
-    ":authority", "authorization", "cache-control", "census", "census-bin",
-    "content-disposition", "content-encoding", "content-language",
-    "content-length", "content-location", "content-range", "content-type",
-    "cookie", "date", "deflate", "deflate,gzip", "", "etag", "expect",
-    "expires", "from", "GET", "grpc", "grpc-accept-encoding", "grpc-encoding",
-    "grpc-internal-encoding-request", "grpc-message", "grpc-status",
-    "grpc-timeout", "gzip", "gzip, deflate", "host", "http", "https",
-    "identity", "identity,deflate", "identity,deflate,gzip", "identity,gzip",
-    "if-match", "if-modified-since", "if-none-match", "if-range",
-    "if-unmodified-since", "last-modified", "link", "location", "max-forwards",
-    ":method", ":path", "POST", "proxy-authenticate", "proxy-authorization",
-    "range", "referer", "refresh", "retry-after", ":scheme", "server",
-    "set-cookie", "/", "/index.html", ":status", "strict-transport-security",
-    "te", "trailers", "transfer-encoding", "user-agent", "vary", "via",
-    "www-authenticate"};
+    ":authority", "authorization", "cache-control", "census-bin",
+    "census-binary-bin", "content-disposition", "content-encoding",
+    "content-language", "content-length", "content-location", "content-range",
+    "content-type", "cookie", "date", "deflate", "deflate,gzip", "", "etag",
+    "expect", "expires", "from", "GET", "grpc", "grpc-accept-encoding",
+    "grpc-encoding", "grpc-internal-encoding-request", "grpc-message",
+    "grpc-status", "grpc-timeout", "gzip", "gzip, deflate", "host", "http",
+    "https", "identity", "identity,deflate", "identity,deflate,gzip",
+    "identity,gzip", "if-match", "if-modified-since", "if-none-match",
+    "if-range", "if-unmodified-since", "last-modified", "link", "location",
+    "max-forwards", ":method", ":path", "POST", "proxy-authenticate",
+    "proxy-authorization", "range", "referer", "refresh", "retry-after",
+    ":scheme", "server", "set-cookie", "/", "/index.html", ":status",
+    "strict-transport-security", "te", "trailers", "transfer-encoding",
+    "user-agent", "vary", "via", "www-authenticate"};
 
 const uint8_t grpc_static_accept_encoding_metadata[8] = {0,  29, 26, 30,
                                                          28, 32, 27, 31};
diff --git a/src/core/transport/static_metadata.h b/src/core/transport/static_metadata.h
index 3803a0488b..ef72b802b5 100644
--- a/src/core/transport/static_metadata.h
+++ b/src/core/transport/static_metadata.h
@@ -94,10 +94,10 @@ extern grpc_mdstr grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];
 #define GRPC_MDSTR_AUTHORIZATION (&grpc_static_mdstr_table[20])
 /* "cache-control" */
 #define GRPC_MDSTR_CACHE_CONTROL (&grpc_static_mdstr_table[21])
-/* "census" */
-#define GRPC_MDSTR_CENSUS (&grpc_static_mdstr_table[22])
 /* "census-bin" */
-#define GRPC_MDSTR_CENSUS_BIN (&grpc_static_mdstr_table[23])
+#define GRPC_MDSTR_CENSUS_BIN (&grpc_static_mdstr_table[22])
+/* "census-binary-bin" */
+#define GRPC_MDSTR_CENSUS_BINARY_BIN (&grpc_static_mdstr_table[23])
 /* "content-disposition" */
 #define GRPC_MDSTR_CONTENT_DISPOSITION (&grpc_static_mdstr_table[24])
 /* "content-encoding" */
diff --git a/src/core/transport/transport.c b/src/core/transport/transport.c
index 08d685668c..6e154b629a 100644
--- a/src/core/transport/transport.c
+++ b/src/core/transport/transport.c
@@ -126,6 +126,7 @@ char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx,
 void grpc_transport_stream_op_finish_with_failure(
     grpc_exec_ctx *exec_ctx, grpc_transport_stream_op *op) {
   grpc_exec_ctx_enqueue(exec_ctx, op->recv_message_ready, false, NULL);
+  grpc_exec_ctx_enqueue(exec_ctx, op->recv_initial_metadata_ready, false, NULL);
   grpc_exec_ctx_enqueue(exec_ctx, op->on_complete, false, NULL);
 }
 
diff --git a/src/core/transport/transport.h b/src/core/transport/transport.h
index f5cac77adc..8902c5d2f6 100644
--- a/src/core/transport/transport.h
+++ b/src/core/transport/transport.h
@@ -92,6 +92,8 @@ typedef struct grpc_transport_stream_op {
 
   /** Receive initial metadata from the stream, into provided metadata batch. */
   grpc_metadata_batch *recv_initial_metadata;
+  /** Should be enqueued when initial metadata is ready to be processed. */
+  grpc_closure *recv_initial_metadata_ready;
 
   /** Receive message data from the stream, into provided byte stream. */
   grpc_byte_stream **recv_message;
@@ -103,7 +105,8 @@ typedef struct grpc_transport_stream_op {
   grpc_metadata_batch *recv_trailing_metadata;
 
   /** Should be enqueued when all requested operations (excluding recv_message
-     which has its own closure) in a given batch have been completed. */
+      and recv_initial_metadata which have their own closures) in a given batch
+      have been completed. */
   grpc_closure *on_complete;
 
   /** If != GRPC_STATUS_OK, cancel this stream */