Use static metadata table in metadata.c

11 files changed, 154 insertions, 11 deletions

diff --git a/include/grpc/support/slice.h b/include/grpc/support/slice.h
index 3abb1b7ca1..507cb19f64 100644
--- a/include/grpc/support/slice.h
+++ b/include/grpc/support/slice.h
@@ -144,6 +144,9 @@ gpr_slice gpr_slice_from_copied_string(const char *source);
      memcpy(slice->data, source, len); */
 gpr_slice gpr_slice_from_copied_buffer(const char *source, size_t len);
 
+/* Create a slice pointing to constant memory */
+gpr_slice gpr_slice_from_static_string(const char *source);
+
 /* Return a result slice derived from s, which shares a ref count with s, where
    result.data==s.data+begin, and result.length==end-begin.
    The ref count of s is increased by one.
diff --git a/src/core/support/slice.c b/src/core/support/slice.c
index 53024e88f1..0470533c1c 100644
--- a/src/core/support/slice.c
+++ b/src/core/support/slice.c
@@ -57,6 +57,20 @@ void gpr_slice_unref(gpr_slice slice) {
   }
 }
 
+/* gpr_slice_from_static_string support structure - a refcount that does
+   nothing */
+static void noop_ref_or_unref(void *unused) {}
+
+static gpr_slice_refcount noop_refcount = {noop_ref_or_unref, noop_ref_or_unref};
+
+gpr_slice gpr_slice_from_static_string(const char *s) {
+  gpr_slice slice;
+  slice.refcount = &noop_refcount;
+  slice.data.refcounted.bytes = (gpr_uint8*)s;
+  slice.data.refcounted.length = strlen(s);
+  return slice;
+}
+
 /* gpr_slice_new support structures - we create a refcount object extended
    with the user provided data pointer & destroy function */
 typedef struct new_slice_refcount {
diff --git a/src/core/surface/init.c b/src/core/surface/init.c
index f8cba01cad..04d68620f1 100644
--- a/src/core/surface/init.c
+++ b/src/core/surface/init.c
@@ -93,6 +93,7 @@ void grpc_init(void) {
   gpr_mu_lock(&g_init_mu);
   if (++g_initializations == 1) {
     gpr_time_init();
+    grpc_mdctx_global_init();
     grpc_lb_policy_registry_init(grpc_pick_first_lb_factory_create());
     grpc_register_lb_policy(grpc_pick_first_lb_factory_create());
     grpc_register_lb_policy(grpc_round_robin_lb_factory_create());
@@ -147,6 +148,7 @@ void grpc_shutdown(void) {
         g_all_of_the_plugins[i].destroy();
       }
     }
+    grpc_mdctx_global_shutdown();
   }
   gpr_mu_unlock(&g_init_mu);
 }
diff --git a/src/core/transport/metadata.c b/src/core/transport/metadata.c
index a72dee4399..da52961fc9 100644
--- a/src/core/transport/metadata.c
+++ b/src/core/transport/metadata.c
@@ -31,7 +31,6 @@
  *
  */
 
-#include "src/core/iomgr/sockaddr.h"
 #include "src/core/transport/metadata.h"
 
 #include <assert.h>
@@ -45,6 +44,7 @@
 #include "src/core/profiling/timers.h"
 #include "src/core/support/murmur_hash.h"
 #include "src/core/transport/chttp2/bin_encoder.h"
+#include "src/core/transport/static_metadata.h"
 
 #define INITIAL_STRTAB_CAPACITY 4
 #define INITIAL_MDTAB_CAPACITY 4
@@ -52,14 +52,14 @@
 #ifdef GRPC_METADATA_REFCOUNT_DEBUG
 #define DEBUG_ARGS , const char *file, int line
 #define FWD_DEBUG_ARGS , file, line
-#define INTERNAL_STRING_REF(s) internal_string_ref((s), __FILE__, __LINE__)
-#define INTERNAL_STRING_UNREF(s) internal_string_unref((s), __FILE__, __LINE__)
+#define INTERNAL_STRING_REF(s) if (is_mdstr_static((grpc_mdstr*)(s))); else internal_string_ref((s), __FILE__, __LINE__)
+#define INTERNAL_STRING_UNREF(s) if (is_mdstr_static((grpc_mdstr*)(s))); else internal_string_unref((s), __FILE__, __LINE__)
 #define REF_MD_LOCKED(s) ref_md_locked((s), __FILE__, __LINE__)
 #else
 #define DEBUG_ARGS
 #define FWD_DEBUG_ARGS
-#define INTERNAL_STRING_REF(s) internal_string_ref((s))
-#define INTERNAL_STRING_UNREF(s) internal_string_unref((s))
+#define INTERNAL_STRING_REF(s) if (is_mdstr_static((grpc_mdstr*)(s))); else internal_string_ref((s))
+#define INTERNAL_STRING_UNREF(s) if (is_mdstr_static((grpc_mdstr*)(s))); else internal_string_unref((s))
 #define REF_MD_LOCKED(s) ref_md_locked((s))
 #endif
 
@@ -98,12 +98,27 @@ typedef struct internal_metadata {
   struct internal_metadata *bucket_next;
 } internal_metadata;
 
+typedef struct static_string {
+  grpc_mdstr *mdstr;
+  gpr_uint32 hash;
+} static_string;
+
+typedef struct static_mdelem {
+  grpc_mdelem *mdelem;
+  gpr_uint32 hash;
+} static_mdelem;
+
 struct grpc_mdctx {
   gpr_uint32 hash_seed;
   int refs;
 
   gpr_mu mu;
 
+  static_string static_strtab[GRPC_STATIC_MDSTR_COUNT * 2];
+  static_mdelem static_mdtab[GRPC_STATIC_MDELEM_COUNT * 2];
+  size_t static_strtab_maxprobe;
+  size_t static_mdtab_maxprobe;
+
   internal_string **strtab;
   size_t strtab_count;
   size_t strtab_capacity;
@@ -120,6 +135,34 @@ static void discard_metadata(grpc_mdctx *ctx);
 static void gc_mdtab(grpc_mdctx *ctx);
 static void metadata_context_destroy_locked(grpc_mdctx *ctx);
 
+void grpc_mdctx_global_init(void) {
+  size_t i;
+  for (i = 0; i < GRPC_STATIC_MDSTR_COUNT; i++) {
+    grpc_mdstr *elem = &grpc_static_mdstr_table[i];
+    const char *str = grpc_static_metadata_strings[i];
+    *(gpr_slice*)&elem->slice = gpr_slice_from_static_string(str);
+    *(gpr_uint32*)&elem->hash = gpr_murmur_hash3(str, strlen(str), 0);
+  }
+  for (i = 0; i < GRPC_STATIC_MDELEM_COUNT; i++) {
+    grpc_mdelem *elem = &grpc_static_mdelem_table[i];
+    grpc_mdstr *key = &grpc_static_mdstr_table[2 * i + 0];
+    grpc_mdstr *value = &grpc_static_mdstr_table[2 * i + 1];
+    *(grpc_mdstr**)&elem->key = key;
+    *(grpc_mdstr**)&elem->value = value;
+  }
+}
+
+void grpc_mdctx_global_shutdown(void) {
+}
+
+static int is_mdstr_static(grpc_mdstr *s) {
+  return s >= &grpc_static_mdstr_table[0] && s < &grpc_static_mdstr_table[GRPC_STATIC_MDSTR_COUNT];
+}
+
+static int is_mdelem_static(grpc_mdelem *e) {
+  return e >= &grpc_static_mdelem_table[0] && e < &grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT];
+}
+
 static void lock(grpc_mdctx *ctx) { gpr_mu_lock(&ctx->mu); }
 
 static void unlock(grpc_mdctx *ctx) {
@@ -170,6 +213,9 @@ static void ref_md_locked(internal_metadata *md DEBUG_ARGS) {
 
 grpc_mdctx *grpc_mdctx_create_with_seed(gpr_uint32 seed) {
   grpc_mdctx *ctx = gpr_malloc(sizeof(grpc_mdctx));
+  size_t i, j;
+
+  memset(ctx, 0, sizeof(*ctx));
 
   ctx->refs = 1;
   ctx->hash_seed = seed;
@@ -184,6 +230,38 @@ grpc_mdctx *grpc_mdctx_create_with_seed(gpr_uint32 seed) {
   ctx->mdtab_capacity = INITIAL_MDTAB_CAPACITY;
   ctx->mdtab_free = 0;
 
+  for (i = 0; i < GRPC_STATIC_MDSTR_COUNT; i++) {
+    const char *str = grpc_static_metadata_strings[i];
+    gpr_uint32 lup_hash = gpr_murmur_hash3(str, strlen(str), seed);
+    for (j = 0;; j++) {
+      size_t idx = (lup_hash + j) % GPR_ARRAY_SIZE(ctx->static_strtab);
+      if (ctx->static_strtab[idx].mdstr == NULL) {
+        ctx->static_strtab[idx].mdstr = &grpc_static_mdstr_table[i];
+        ctx->static_strtab[idx].hash = lup_hash;
+        break;
+      }
+    }
+    if (j > ctx->static_strtab_maxprobe) {
+      ctx->static_strtab_maxprobe = j;
+    }
+  }
+
+  for (i = 0; i < GRPC_STATIC_MDELEM_COUNT; i++) {
+    grpc_mdelem *elem = &grpc_static_mdelem_table[i];
+    gpr_uint32 hash = GRPC_MDSTR_KV_HASH(elem->key->hash, elem->value->hash);
+    for (j = 0;; j++) {
+      size_t idx = (hash + j) % GPR_ARRAY_SIZE(ctx->static_mdtab);
+      if (ctx->static_mdtab[idx].mdelem == NULL) {
+        ctx->static_mdtab[idx].mdelem = elem;
+        ctx->static_mdtab[idx].hash = hash;
+        break;
+      }
+    }
+    if (j > ctx->static_mdtab_maxprobe) {
+      ctx->static_mdtab_maxprobe = j;
+    }
+  }
+
   return ctx;
 }
 
@@ -350,8 +428,20 @@ grpc_mdstr *grpc_mdstr_from_buffer(grpc_mdctx *ctx, const gpr_uint8 *buf,
                                    size_t length) {
   gpr_uint32 hash = gpr_murmur_hash3(buf, length, ctx->hash_seed);
   internal_string *s;
+  size_t i;
 
   GPR_TIMER_BEGIN("grpc_mdstr_from_buffer", 0);
+
+  /* search for a static string */
+  for (i = 0; i <= ctx->static_strtab_maxprobe; i++) {
+    size_t idx = (hash + i) % GPR_ARRAY_SIZE(ctx->static_strtab);
+    static_string *ss = &ctx->static_strtab[idx];
+    if (ss->hash == hash && GPR_SLICE_LENGTH(ss->mdstr->slice) == length &&
+        0 == memcmp(buf, GPR_SLICE_START_PTR(ss->mdstr->slice), length)) {
+      return ss->mdstr;
+    }
+  }
+
   lock(ctx);
 
   /* search for an existing string */
@@ -479,12 +569,23 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdctx *ctx,
   internal_string *value = (internal_string *)mvalue;
   gpr_uint32 hash = GRPC_MDSTR_KV_HASH(mkey->hash, mvalue->hash);
   internal_metadata *md;
+  size_t i;
 
-  GPR_ASSERT(key->context == ctx);
-  GPR_ASSERT(value->context == ctx);
+  GPR_ASSERT(is_mdstr_static(mkey) || key->context == ctx);
+  GPR_ASSERT(is_mdstr_static(mvalue) || value->context == ctx);
 
   GPR_TIMER_BEGIN("grpc_mdelem_from_metadata_strings", 0);
 
+  if (is_mdstr_static(mkey) && is_mdstr_static(mvalue)) {
+    for (i = 0; i <= ctx->static_mdtab_maxprobe; i++) {
+      size_t idx = (hash + i) % GPR_ARRAY_SIZE(ctx->static_mdtab);
+      static_mdelem *smd = &ctx->static_mdtab[idx];
+      if (smd->hash == hash && smd->mdelem->key == mkey && smd->mdelem->value == mvalue) {
+        return smd->mdelem;
+      }
+    }
+  }
+
   lock(ctx);
 
   /* search for an existing pair */
@@ -553,6 +654,7 @@ grpc_mdelem *grpc_mdelem_from_string_and_buffer(grpc_mdctx *ctx,
 
 grpc_mdelem *grpc_mdelem_ref(grpc_mdelem *gmd DEBUG_ARGS) {
   internal_metadata *md = (internal_metadata *)gmd;
+  if (is_mdelem_static(gmd)) return gmd;
 #ifdef GRPC_METADATA_REFCOUNT_DEBUG
   gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
           "ELM   REF:%p:%d->%d: '%s' = '%s'", md,
@@ -573,6 +675,7 @@ grpc_mdelem *grpc_mdelem_ref(grpc_mdelem *gmd DEBUG_ARGS) {
 void grpc_mdelem_unref(grpc_mdelem *gmd DEBUG_ARGS) {
   internal_metadata *md = (internal_metadata *)gmd;
   if (!md) return;
+  if (is_mdelem_static(gmd)) return;
 #ifdef GRPC_METADATA_REFCOUNT_DEBUG
   gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
           "ELM UNREF:%p:%d->%d: '%s' = '%s'", md,
@@ -600,7 +703,9 @@ const char *grpc_mdstr_as_c_string(grpc_mdstr *s) {
 
 grpc_mdstr *grpc_mdstr_ref(grpc_mdstr *gs DEBUG_ARGS) {
   internal_string *s = (internal_string *)gs;
-  grpc_mdctx *ctx = s->context;
+  grpc_mdctx *ctx;
+  if (is_mdstr_static(gs)) return gs;
+  ctx = s->context;
   lock(ctx);
   internal_string_ref(s FWD_DEBUG_ARGS);
   unlock(ctx);
@@ -609,7 +714,9 @@ grpc_mdstr *grpc_mdstr_ref(grpc_mdstr *gs DEBUG_ARGS) {
 
 void grpc_mdstr_unref(grpc_mdstr *gs DEBUG_ARGS) {
   internal_string *s = (internal_string *)gs;
-  grpc_mdctx *ctx = s->context;
+  grpc_mdctx *ctx;
+  if (is_mdstr_static(gs)) return;
+  ctx = s->context;
   lock(ctx);
   internal_string_unref(s FWD_DEBUG_ARGS);
   unlock(ctx);
diff --git a/src/core/transport/metadata.h b/src/core/transport/metadata.h
index 9a8164037c..4c52896299 100644
--- a/src/core/transport/metadata.h
+++ b/src/core/transport/metadata.h
@@ -157,4 +157,7 @@ int grpc_mdstr_is_bin_suffixed(grpc_mdstr *s);
 
 #define GRPC_MDSTR_KV_HASH(k_hash, v_hash) (GPR_ROTL((k_hash), 2) ^ (v_hash))
 
+void grpc_mdctx_global_init(void);
+void grpc_mdctx_global_shutdown(void);
+
 #endif /* GRPC_INTERNAL_CORE_TRANSPORT_METADATA_H */
diff --git a/test/core/bad_client/bad_client.c b/test/core/bad_client/bad_client.c
index ed46e7b009..fb2cd2d85b 100644
--- a/test/core/bad_client/bad_client.c
+++ b/test/core/bad_client/bad_client.c
@@ -84,7 +84,7 @@ void grpc_run_bad_client_test(grpc_bad_client_server_side_validator validator,
   gpr_thd_id id;
   char *hex;
   grpc_transport *transport;
-  grpc_mdctx *mdctx = grpc_mdctx_create();
+  grpc_mdctx *mdctx;
   gpr_slice slice =
       gpr_slice_from_copied_buffer(client_payload, client_payload_length);
   gpr_slice_buffer outgoing;
@@ -102,6 +102,8 @@ void grpc_run_bad_client_test(grpc_bad_client_server_side_validator validator,
   /* Init grpc */
   grpc_init();
 
+  mdctx = grpc_mdctx_create();
+
   /* Create endpoints */
   sfd = grpc_iomgr_create_endpoint_pair("fixture", 65536);
 
diff --git a/test/core/channel/channel_stack_test.c b/test/core/channel/channel_stack_test.c
index 5e1ba11857..08550b4934 100644
--- a/test/core/channel/channel_stack_test.c
+++ b/test/core/channel/channel_stack_test.c
@@ -140,6 +140,8 @@ static void test_create_channel_stack(void) {
 
 int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
+  grpc_init();
   test_create_channel_stack();
+  grpc_shutdown();
   return 0;
 }
diff --git a/test/core/transport/chttp2/hpack_encoder_test.c b/test/core/transport/chttp2/hpack_encoder_test.c
index 6553e0da20..30cb6c1d17 100644
--- a/test/core/transport/chttp2/hpack_encoder_test.c
+++ b/test/core/transport/chttp2/hpack_encoder_test.c
@@ -191,8 +191,10 @@ static void run_test(void (*test)(), const char *name) {
 int main(int argc, char **argv) {
   size_t i;
   grpc_test_init(argc, argv);
+  grpc_init();
   TEST(test_basic_headers);
   TEST(test_decode_table_overflow);
+  grpc_shutdown();
   for (i = 0; i < num_to_delete; i++) {
     gpr_free(to_delete[i]);
   }
diff --git a/test/core/transport/chttp2/hpack_parser_test.c b/test/core/transport/chttp2/hpack_parser_test.c
index 3a313375a4..4e52b0e466 100644
--- a/test/core/transport/chttp2/hpack_parser_test.c
+++ b/test/core/transport/chttp2/hpack_parser_test.c
@@ -217,7 +217,9 @@ static void test_vectors(grpc_slice_split_mode mode) {
 
 int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
+  grpc_init();
   test_vectors(GRPC_SLICE_SPLIT_MERGE_ALL);
   test_vectors(GRPC_SLICE_SPLIT_ONE_BYTE);
+  grpc_shutdown();
   return 0;
 }
diff --git a/test/core/transport/chttp2/hpack_table_test.c b/test/core/transport/chttp2/hpack_table_test.c
index aa3e273a6c..adc69bf314 100644
--- a/test/core/transport/chttp2/hpack_table_test.c
+++ b/test/core/transport/chttp2/hpack_table_test.c
@@ -36,10 +36,12 @@
 #include <string.h>
 #include <stdio.h>
 
-#include "src/core/support/string.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
+#include <grpc/grpc.h>
+
+#include "src/core/support/string.h"
 #include "test/core/util/test_config.h"
 
 #define LOG_TEST(x) gpr_log(GPR_INFO, "%s", x)
@@ -268,8 +270,10 @@ static void test_find(void) {
 
 int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
+  grpc_init();
   test_static_lookup();
   test_many_additions();
   test_find();
+  grpc_shutdown();
   return 0;
 }
diff --git a/test/core/transport/metadata_test.c b/test/core/transport/metadata_test.c
index 080e86cb63..fc9c93a774 100644
--- a/test/core/transport/metadata_test.c
+++ b/test/core/transport/metadata_test.c
@@ -270,6 +270,7 @@ static void test_base64_and_huffman_works(void) {
 
 int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
+  grpc_init();
   test_no_op();
   test_create_string();
   test_create_metadata();
@@ -279,5 +280,6 @@ int main(int argc, char **argv) {
   test_things_stick_around();
   test_slices_work();
   test_base64_and_huffman_works();
+  grpc_shutdown();
   return 0;
 }