Optimize framing a little

- rely on the fact that data-to-come holds a reference to
data-being-written, so there's no need to add a ref for every frame
written
- provide an 'inlined' version of grpc_slice_malloc (via a #define) that
gives the compiler more information about small allocations to enable
better optimization

1 files changed, 68 insertions, 27 deletions

diff --git a/src/core/lib/slice/slice.c b/src/core/lib/slice/slice.c
index 1cddf062cd..8896f1dfd3 100644
--- a/src/core/lib/slice/slice.c
+++ b/src/core/lib/slice/slice.c
@@ -197,7 +197,7 @@ grpc_slice grpc_slice_new_with_len(void *p, size_t len,
 }
 
 grpc_slice grpc_slice_from_copied_buffer(const char *source, size_t length) {
-  grpc_slice slice = grpc_slice_malloc(length);
+  grpc_slice slice = GRPC_SLICE_MALLOC(length);
   memcpy(GRPC_SLICE_START_PTR(slice), source, length);
   return slice;
 }
@@ -227,35 +227,42 @@ static const grpc_slice_refcount_vtable malloc_vtable = {
     malloc_ref, malloc_unref, grpc_slice_default_eq_impl,
     grpc_slice_default_hash_impl};
 
+grpc_slice grpc_slice_malloc_large(size_t length) {
+  grpc_slice slice;
+
+  /* Memory layout used by the slice created here:
+
+     +-----------+----------------------------------------------------------+
+     | refcount  | bytes                                                    |
+     +-----------+----------------------------------------------------------+
+
+     refcount is a malloc_refcount
+     bytes is an array of bytes of the requested length
+     Both parts are placed in the same allocation returned from gpr_malloc */
+  malloc_refcount *rc = gpr_malloc(sizeof(malloc_refcount) + length);
+
+  /* Initial refcount on rc is 1 - and it's up to the caller to release
+     this reference. */
+  gpr_ref_init(&rc->refs, 1);
+
+  rc->base.vtable = &malloc_vtable;
+  rc->base.sub_refcount = &rc->base;
+
+  /* Build up the slice to be returned. */
+  /* The slices refcount points back to the allocated block. */
+  slice.refcount = &rc->base;
+  /* The data bytes are placed immediately after the refcount struct */
+  slice.data.refcounted.bytes = (uint8_t *)(rc + 1);
+  /* And the length of the block is set to the requested length */
+  slice.data.refcounted.length = length;
+  return slice;
+}
+
 grpc_slice grpc_slice_malloc(size_t length) {
   grpc_slice slice;
 
   if (length > sizeof(slice.data.inlined.bytes)) {
-    /* Memory layout used by the slice created here:
-
-       +-----------+----------------------------------------------------------+
-       | refcount  | bytes                                                    |
-       +-----------+----------------------------------------------------------+
-
-       refcount is a malloc_refcount
-       bytes is an array of bytes of the requested length
-       Both parts are placed in the same allocation returned from gpr_malloc */
-    malloc_refcount *rc = gpr_malloc(sizeof(malloc_refcount) + length);
-
-    /* Initial refcount on rc is 1 - and it's up to the caller to release
-       this reference. */
-    gpr_ref_init(&rc->refs, 1);
-
-    rc->base.vtable = &malloc_vtable;
-    rc->base.sub_refcount = &rc->base;
-
-    /* Build up the slice to be returned. */
-    /* The slices refcount points back to the allocated block. */
-    slice.refcount = &rc->base;
-    /* The data bytes are placed immediately after the refcount struct */
-    slice.data.refcounted.bytes = (uint8_t *)(rc + 1);
-    /* And the length of the block is set to the requested length */
-    slice.data.refcounted.length = length;
+    return grpc_slice_malloc_large(length);
   } else {
     /* small slice: just inline the data */
     slice.refcount = NULL;
@@ -341,6 +348,40 @@ grpc_slice grpc_slice_split_tail(grpc_slice *source, size_t split) {
   return tail;
 }
 
+grpc_slice grpc_slice_split_tail_no_ref(grpc_slice *source, size_t split) {
+  grpc_slice tail;
+
+  if (source->refcount == NULL) {
+    /* inlined data, copy it out */
+    GPR_ASSERT(source->data.inlined.length >= split);
+    tail.refcount = NULL;
+    tail.data.inlined.length = (uint8_t)(source->data.inlined.length - split);
+    memcpy(tail.data.inlined.bytes, source->data.inlined.bytes + split,
+           tail.data.inlined.length);
+    source->data.inlined.length = (uint8_t)split;
+  } else {
+    size_t tail_length = source->data.refcounted.length - split;
+    GPR_ASSERT(source->data.refcounted.length >= split);
+    if (tail_length < sizeof(tail.data.inlined.bytes)) {
+      /* Copy out the bytes - it'll be cheaper than refcounting */
+      tail.refcount = NULL;
+      tail.data.inlined.length = (uint8_t)tail_length;
+      memcpy(tail.data.inlined.bytes, source->data.refcounted.bytes + split,
+             tail_length);
+    } else {
+      /* Build the result */
+      tail.refcount = &noop_refcount;
+      /* Point into the source array */
+      tail.data.refcounted.bytes = source->data.refcounted.bytes + split;
+      tail.data.refcounted.length = tail_length;
+    }
+    source->refcount = source->refcount->sub_refcount;
+    source->data.refcounted.length = split;
+  }
+
+  return tail;
+}
+
 grpc_slice grpc_slice_split_head(grpc_slice *source, size_t split) {
   grpc_slice head;
 
@@ -457,7 +498,7 @@ int grpc_slice_slice(grpc_slice haystack, grpc_slice needle) {
 }
 
 grpc_slice grpc_slice_dup(grpc_slice a) {
-  grpc_slice copy = grpc_slice_malloc(GRPC_SLICE_LENGTH(a));
+  grpc_slice copy = GRPC_SLICE_MALLOC(GRPC_SLICE_LENGTH(a));
   memcpy(GRPC_SLICE_START_PTR(copy), GRPC_SLICE_START_PTR(a),
          GRPC_SLICE_LENGTH(a));
   return copy;