diff options
Diffstat (limited to 'src')
24 files changed, 255 insertions, 163 deletions
diff --git a/src/core/ext/census/grpc_filter.c b/src/core/ext/census/grpc_filter.c index c1a883b6f8..6e31994280 100644 --- a/src/core/ext/census/grpc_filter.c +++ b/src/core/ext/census/grpc_filter.c @@ -67,7 +67,7 @@ static void extract_and_annotate_method_tag(grpc_metadata_batch *md, channel_data *chand) { grpc_linked_mdelem *m; for (m = md->list.head; m != NULL; m = m->next) { - if (grpc_slice_cmp(GRPC_MDKEY(m->md), GRPC_MDSTR_PATH) == 0) { + if (grpc_slice_eq(GRPC_MDKEY(m->md), GRPC_MDSTR_PATH)) { /* Add method tag here */ } } diff --git a/src/core/ext/load_reporting/load_reporting_filter.c b/src/core/ext/load_reporting/load_reporting_filter.c index 5d3b4c4207..75f5c73ae3 100644 --- a/src/core/ext/load_reporting/load_reporting_filter.c +++ b/src/core/ext/load_reporting/load_reporting_filter.c @@ -78,10 +78,10 @@ static grpc_mdelem recv_md_filter(grpc_exec_ctx *exec_ctx, void *user_data, grpc_call_element *elem = a->elem; call_data *calld = elem->call_data; - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_PATH) == 0) { + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_PATH)) { calld->service_method = grpc_slice_ref_internal(GRPC_MDVALUE(md)); calld->have_service_method = true; - } else if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_LB_TOKEN) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_LB_TOKEN)) { calld->initial_md_string = grpc_slice_ref_internal(GRPC_MDVALUE(md)); calld->have_initial_md_string = true; return GRPC_MDNULL; @@ -201,7 +201,7 @@ static grpc_mdelem lr_trailing_md_filter(grpc_exec_ctx *exec_ctx, grpc_call_element *elem = user_data; call_data *calld = elem->call_data; - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_LB_COST_BIN) == 0) { + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_LB_COST_BIN)) { calld->trailing_md_string = grpc_slice_ref_internal(GRPC_MDVALUE(md)); calld->have_trailing_md_string = true; return GRPC_MDNULL; diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 91861829e3..586052fdd3 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -876,7 +876,7 @@ void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx, static bool contains_non_ok_status(grpc_metadata_batch *batch) { grpc_linked_mdelem *l; for (l = batch->list.head; l; l = l->next) { - if (grpc_slice_cmp(GRPC_MDKEY(l->md), GRPC_MDSTR_GRPC_STATUS) == 0 && + if (grpc_slice_eq(GRPC_MDKEY(l->md), GRPC_MDSTR_GRPC_STATUS) && !grpc_mdelem_eq(l->md, GRPC_MDELEM_GRPC_STATUS_0)) { return true; } diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 7f7dfa5b55..eb19bd7c2d 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -49,6 +49,7 @@ #include "src/core/ext/transport/chttp2/transport/hpack_table.h" #include "src/core/ext/transport/chttp2/transport/varint.h" #include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/transport/metadata.h" #include "src/core/lib/transport/static_metadata.h" #include "src/core/lib/transport/timeout_encoding.h" @@ -187,22 +188,10 @@ static void evict_entry(grpc_chttp2_hpack_compressor *c) { c->table_elems--; } -static bool is_interned(grpc_mdelem elem) { - switch (GRPC_MDELEM_STORAGE(elem)) { - case GRPC_MDELEM_STORAGE_ALLOCATED: - case GRPC_MDELEM_STORAGE_EXTERNAL: - return false; - case GRPC_MDELEM_STORAGE_INTERNED: - case GRPC_MDELEM_STORAGE_STATIC: - return true; - } - GPR_UNREACHABLE_CODE(return false); -} - /* add an element to the decoder table */ static void add_elem(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, grpc_mdelem elem) { - GPR_ASSERT(is_interned(elem)); + GPR_ASSERT(GRPC_MDELEM_IS_INTERNED(elem)); uint32_t key_hash = grpc_slice_hash(GRPC_MDKEY(elem)); uint32_t value_hash = grpc_slice_hash(GRPC_MDVALUE(elem)); @@ -261,11 +250,11 @@ static void add_elem(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, /* do exactly the same for the key (so we can find by that again too) */ - if (grpc_slice_cmp(c->entries_keys[HASH_FRAGMENT_2(key_hash)], - GRPC_MDKEY(elem)) == 0) { + if (grpc_slice_eq(c->entries_keys[HASH_FRAGMENT_2(key_hash)], + GRPC_MDKEY(elem))) { c->indices_keys[HASH_FRAGMENT_2(key_hash)] = new_index; - } else if (grpc_slice_cmp(c->entries_keys[HASH_FRAGMENT_3(key_hash)], - GRPC_MDKEY(elem)) == 0) { + } else if (grpc_slice_eq(c->entries_keys[HASH_FRAGMENT_3(key_hash)], + GRPC_MDKEY(elem))) { c->indices_keys[HASH_FRAGMENT_3(key_hash)] = new_index; } else if (c->entries_keys[HASH_FRAGMENT_2(key_hash)].refcount == &terminal_slice_refcount) { @@ -407,7 +396,19 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, "Reserved header (colon-prefixed) happening after regular ones."); } - if (!is_interned(elem)) { + if (grpc_http_trace && !GRPC_MDELEM_IS_INTERNED(elem)) { + char *k = grpc_dump_slice(GRPC_MDKEY(elem), GPR_DUMP_ASCII); + char *v = grpc_dump_slice(GRPC_MDVALUE(elem), GPR_DUMP_ASCII); + gpr_log( + GPR_DEBUG, + "Encode: '%s: %s', elem_interned=%d [%d], k_interned=%d, v_interned=%d", + k, v, GRPC_MDELEM_IS_INTERNED(elem), GRPC_MDELEM_STORAGE(elem), + grpc_slice_is_interned(GRPC_MDKEY(elem)), + grpc_slice_is_interned(GRPC_MDVALUE(elem))); + gpr_free(k); + gpr_free(v); + } + if (!GRPC_MDELEM_IS_INTERNED(elem)) { emit_lithdr_noidx_v(c, elem, st); return; } @@ -452,8 +453,8 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, /* no hits for the elem... maybe there's a key? */ indices_key = c->indices_keys[HASH_FRAGMENT_2(key_hash)]; - if (grpc_slice_cmp(c->entries_keys[HASH_FRAGMENT_2(key_hash)], - GRPC_MDKEY(elem)) == 0 && + if (grpc_slice_eq(c->entries_keys[HASH_FRAGMENT_2(key_hash)], + GRPC_MDKEY(elem)) && indices_key > c->tail_remote_index) { /* HIT: key (first cuckoo hash) */ if (should_add_elem) { @@ -468,8 +469,8 @@ static void hpack_enc(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_compressor *c, } indices_key = c->indices_keys[HASH_FRAGMENT_3(key_hash)]; - if (grpc_slice_cmp(c->entries_keys[HASH_FRAGMENT_3(key_hash)], - GRPC_MDKEY(elem)) == 0 && + if (grpc_slice_eq(c->entries_keys[HASH_FRAGMENT_3(key_hash)], + GRPC_MDKEY(elem)) && indices_key > c->tail_remote_index) { /* HIT: key (first cuckoo hash) */ if (should_add_elem) { diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index a1ff528d72..c53a1faee3 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -55,6 +55,9 @@ #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/support/string.h" +/* TODO(ctiller): remove before submission */ +#include "src/core/lib/slice/slice_string_helpers.h" + extern int grpc_http_trace; typedef enum { @@ -670,6 +673,18 @@ static const uint8_t inverse_base64[256] = { /* emission helpers */ static grpc_error *on_hdr(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, grpc_mdelem md, int add_to_table) { + if (!GRPC_MDELEM_IS_INTERNED(md)) { + char *k = grpc_dump_slice(GRPC_MDKEY(md), GPR_DUMP_ASCII); + char *v = grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_ASCII); + gpr_log( + GPR_DEBUG, + "Decode: '%s: %s', elem_interned=%d [%d], k_interned=%d, v_interned=%d", + k, v, GRPC_MDELEM_IS_INTERNED(md), GRPC_MDELEM_STORAGE(md), + grpc_slice_is_interned(GRPC_MDKEY(md)), + grpc_slice_is_interned(GRPC_MDVALUE(md))); + gpr_free(k); + gpr_free(v); + } if (add_to_table) { GPR_ASSERT(GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_INTERNED || GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC); @@ -684,16 +699,28 @@ static grpc_error *on_hdr(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, return GRPC_ERROR_NONE; } -static grpc_slice take_string(grpc_chttp2_hpack_parser *p, +static grpc_slice take_string(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, grpc_chttp2_hpack_parser_string *str, bool intern) { grpc_slice s; - if (intern) { - s = grpc_slice_intern(grpc_slice_from_static_buffer(str->str, str->length)); + if (!str->copied) { + if (intern) { + s = grpc_slice_intern(str->data.referenced); + grpc_slice_unref_internal(exec_ctx, str->data.referenced); + } else { + s = str->data.referenced; + } + str->copied = true; + str->data.referenced = grpc_empty_slice(); + } else if (intern) { + s = grpc_slice_intern(grpc_slice_from_static_buffer( + str->data.copied.str, str->data.copied.length)); } else { - s = grpc_slice_from_copied_buffer(str->str, str->length); + s = grpc_slice_from_copied_buffer(str->data.copied.str, + str->data.copied.length); } - str->length = 0; + str->data.copied.length = 0; return s; } @@ -827,7 +854,7 @@ static grpc_error *finish_lithdr_incidx(grpc_exec_ctx *exec_ctx, grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, grpc_slice_ref_internal(GRPC_MDKEY(md)), - take_string(p, &p->value, true)), + take_string(exec_ctx, p, &p->value, true)), 1); if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); return parse_begin(exec_ctx, p, cur, end); @@ -838,11 +865,11 @@ static grpc_error *finish_lithdr_incidx_v(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { - grpc_error *err = - on_hdr(exec_ctx, p, - grpc_mdelem_from_slices(exec_ctx, take_string(p, &p->key, true), - take_string(p, &p->value, true)), - 1); + grpc_error *err = on_hdr( + exec_ctx, p, + grpc_mdelem_from_slices(exec_ctx, take_string(exec_ctx, p, &p->key, true), + take_string(exec_ctx, p, &p->value, true)), + 1); if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); return parse_begin(exec_ctx, p, cur, end); } @@ -897,7 +924,7 @@ static grpc_error *finish_lithdr_notidx(grpc_exec_ctx *exec_ctx, grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, grpc_slice_ref_internal(GRPC_MDKEY(md)), - take_string(p, &p->value, false)), + take_string(exec_ctx, p, &p->value, false)), 0); if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); return parse_begin(exec_ctx, p, cur, end); @@ -908,11 +935,11 @@ static grpc_error *finish_lithdr_notidx_v(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { - grpc_error *err = - on_hdr(exec_ctx, p, - grpc_mdelem_from_slices(exec_ctx, take_string(p, &p->key, false), - take_string(p, &p->value, false)), - 0); + grpc_error *err = on_hdr( + exec_ctx, p, grpc_mdelem_from_slices( + exec_ctx, take_string(exec_ctx, p, &p->key, false), + take_string(exec_ctx, p, &p->value, false)), + 0); if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); return parse_begin(exec_ctx, p, cur, end); } @@ -967,7 +994,7 @@ static grpc_error *finish_lithdr_nvridx(grpc_exec_ctx *exec_ctx, grpc_error *err = on_hdr( exec_ctx, p, grpc_mdelem_from_slices(exec_ctx, grpc_slice_ref_internal(GRPC_MDKEY(md)), - take_string(p, &p->value, false)), + take_string(exec_ctx, p, &p->value, false)), 0); if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); return parse_begin(exec_ctx, p, cur, end); @@ -978,11 +1005,11 @@ static grpc_error *finish_lithdr_nvridx_v(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { - grpc_error *err = - on_hdr(exec_ctx, p, - grpc_mdelem_from_slices(exec_ctx, take_string(p, &p->key, false), - take_string(p, &p->value, false)), - 0); + grpc_error *err = on_hdr( + exec_ctx, p, grpc_mdelem_from_slices( + exec_ctx, take_string(exec_ctx, p, &p->key, false), + take_string(exec_ctx, p, &p->value, false)), + 0); if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); return parse_begin(exec_ctx, p, cur, end); } @@ -1276,14 +1303,15 @@ static grpc_error *parse_string_prefix(grpc_exec_ctx *exec_ctx, static void append_bytes(grpc_chttp2_hpack_parser_string *str, const uint8_t *data, size_t length) { if (length == 0) return; - if (length + str->length > str->capacity) { - GPR_ASSERT(str->length + length <= UINT32_MAX); - str->capacity = (uint32_t)(str->length + length); - str->str = gpr_realloc(str->str, str->capacity); + if (length + str->data.copied.length > str->data.copied.capacity) { + GPR_ASSERT(str->data.copied.length + length <= UINT32_MAX); + str->data.copied.capacity = (uint32_t)(str->data.copied.length + length); + str->data.copied.str = + gpr_realloc(str->data.copied.str, str->data.copied.capacity); } - memcpy(str->str + str->length, data, length); - GPR_ASSERT(length <= UINT32_MAX - str->length); - str->length += (uint32_t)length; + memcpy(str->data.copied.str + str->data.copied.length, data, length); + GPR_ASSERT(length <= UINT32_MAX - str->data.copied.length); + str->data.copied.length += (uint32_t)length; } static grpc_error *append_string(grpc_exec_ctx *exec_ctx, @@ -1366,11 +1394,9 @@ static grpc_error *append_string(grpc_exec_ctx *exec_ctx, exec_ctx, p, cur, end, GRPC_ERROR_CREATE("Should never reach here"))); } -/* append a null terminator to a string */ static grpc_error *finish_str(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { - uint8_t terminator = 0; uint8_t decoded[2]; uint32_t bits; grpc_chttp2_hpack_parser_string *str = p->parsing.str; @@ -1411,8 +1437,6 @@ static grpc_error *finish_str(grpc_exec_ctx *exec_ctx, append_bytes(str, decoded, 2); break; } - append_bytes(str, &terminator, 1); - p->parsing.str->length--; /* don't actually count the null terminator */ return GRPC_ERROR_NONE; } @@ -1487,8 +1511,18 @@ static grpc_error *begin_parse_string(grpc_exec_ctx *exec_ctx, const uint8_t *cur, const uint8_t *end, uint8_t binary, grpc_chttp2_hpack_parser_string *str) { + if (!p->huff && binary == NOT_BINARY && (end - cur) >= p->strlen && + p->current_slice_refcount != NULL) { + str->copied = false; + str->data.referenced.refcount = p->current_slice_refcount; + str->data.referenced.data.refcounted.bytes = (uint8_t *)cur; + str->data.referenced.data.refcounted.length = p->strlen; + grpc_slice_ref_internal(str->data.referenced); + return parse_next(exec_ctx, p, cur + p->strlen, end); + } p->strgot = 0; - str->length = 0; + str->copied = true; + str->data.copied.length = 0; p->parsing.str = str; p->huff_state = 0; p->binary = binary; @@ -1505,8 +1539,8 @@ static grpc_error *parse_key_string(grpc_exec_ctx *exec_ctx, /* check if a key represents a binary header or not */ static bool is_binary_literal_header(grpc_chttp2_hpack_parser *p) { - return grpc_is_binary_header( - grpc_slice_from_static_buffer(p->key.str, p->key.length)); + return grpc_is_binary_header(grpc_slice_from_static_buffer( + p->key.data.copied.str, p->key.data.copied.length)); } static grpc_error *is_binary_indexed_header(grpc_chttp2_hpack_parser *p, @@ -1553,12 +1587,12 @@ void grpc_chttp2_hpack_parser_init(grpc_exec_ctx *exec_ctx, p->on_header = NULL; p->on_header_user_data = NULL; p->state = parse_begin; - p->key.str = NULL; - p->key.capacity = 0; - p->key.length = 0; - p->value.str = NULL; - p->value.capacity = 0; - p->value.length = 0; + p->key.data.copied.str = NULL; + p->key.data.copied.capacity = 0; + p->key.data.copied.length = 0; + p->value.data.copied.str = NULL; + p->value.data.copied.capacity = 0; + p->value.data.copied.length = 0; p->dynamic_table_update_allowed = 2; p->last_error = GRPC_ERROR_NONE; grpc_chttp2_hptbl_init(exec_ctx, &p->table); @@ -1573,19 +1607,25 @@ void grpc_chttp2_hpack_parser_destroy(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p) { grpc_chttp2_hptbl_destroy(exec_ctx, &p->table); GRPC_ERROR_UNREF(p->last_error); - gpr_free(p->key.str); - gpr_free(p->value.str); + grpc_slice_unref_internal(exec_ctx, p->key.data.referenced); + grpc_slice_unref_internal(exec_ctx, p->value.data.referenced); + gpr_free(p->key.data.copied.str); + gpr_free(p->value.data.copied.str); } grpc_error *grpc_chttp2_hpack_parser_parse(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, + grpc_slice_refcount *refcount, const uint8_t *beg, const uint8_t *end) { /* TODO(ctiller): limit the distance of end from beg, and perform multiple steps in the event of a large chunk of data to limit stack space usage when no tail call optimization is available */ - return p->state(exec_ctx, p, beg, end); + p->current_slice_refcount = refcount; + grpc_error *error = p->state(exec_ctx, p, beg, end); + p->current_slice_refcount = NULL; + return error; } typedef void (*maybe_complete_func_type)(grpc_exec_ctx *exec_ctx, @@ -1620,7 +1660,8 @@ grpc_error *grpc_chttp2_header_parser_parse(grpc_exec_ctx *exec_ctx, s->stats.incoming.header_bytes += GRPC_SLICE_LENGTH(slice); } grpc_error *error = grpc_chttp2_hpack_parser_parse( - exec_ctx, parser, GRPC_SLICE_START_PTR(slice), GRPC_SLICE_END_PTR(slice)); + exec_ctx, parser, slice.refcount, GRPC_SLICE_START_PTR(slice), + GRPC_SLICE_END_PTR(slice)); if (error != GRPC_ERROR_NONE) { GPR_TIMER_END("grpc_chttp2_hpack_parser_parse", 0); return error; diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.h b/src/core/ext/transport/chttp2/transport/hpack_parser.h index 442708e3d7..3ab0a2a34b 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.h +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.h @@ -49,9 +49,15 @@ typedef grpc_error *(*grpc_chttp2_hpack_parser_state)( const uint8_t *end); typedef struct { - char *str; - uint32_t length; - uint32_t capacity; + bool copied; + struct { + grpc_slice referenced; + struct { + char *str; + uint32_t length; + uint32_t capacity; + } copied; + } data; } grpc_chttp2_hpack_parser_string; struct grpc_chttp2_hpack_parser { @@ -67,6 +73,8 @@ struct grpc_chttp2_hpack_parser { const grpc_chttp2_hpack_parser_state *next_state; /* what to do after skipping prioritization data */ grpc_chttp2_hpack_parser_state after_prioritization; + /* the refcount of the slice that we're currently parsing */ + grpc_slice_refcount *current_slice_refcount; /* the value we're currently parsing */ union { uint32_t *value; @@ -106,9 +114,9 @@ void grpc_chttp2_hpack_parser_destroy(grpc_exec_ctx *exec_ctx, void grpc_chttp2_hpack_parser_set_has_priority(grpc_chttp2_hpack_parser *p); -/* returns 1 on success, 0 on error */ grpc_error *grpc_chttp2_hpack_parser_parse(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, + grpc_slice_refcount *refcount, const uint8_t *beg, const uint8_t *end); diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.c b/src/core/ext/transport/chttp2/transport/hpack_table.c index 2cb816fe53..62dd1b8cab 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_table.c +++ b/src/core/ext/transport/chttp2/transport/hpack_table.c @@ -362,9 +362,9 @@ grpc_chttp2_hptbl_find_result grpc_chttp2_hptbl_find( /* See if the string is in the static table */ for (i = 0; i < GRPC_CHTTP2_LAST_STATIC_ENTRY; i++) { grpc_mdelem ent = tbl->static_ents[i]; - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDKEY(ent)) != 0) continue; + if (!grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDKEY(ent))) continue; r.index = i + 1u; - r.has_value = grpc_slice_cmp(GRPC_MDVALUE(md), GRPC_MDVALUE(ent)) == 0; + r.has_value = grpc_slice_eq(GRPC_MDVALUE(md), GRPC_MDVALUE(ent)); if (r.has_value) return r; } @@ -373,9 +373,9 @@ grpc_chttp2_hptbl_find_result grpc_chttp2_hptbl_find( uint32_t idx = (uint32_t)(tbl->num_ents - i + GRPC_CHTTP2_LAST_STATIC_ENTRY); grpc_mdelem ent = tbl->ents[(tbl->first_ent + i) % tbl->cap_entries]; - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDKEY(ent)) != 0) continue; + if (!grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDKEY(ent))) continue; r.index = idx; - r.has_value = grpc_slice_cmp(GRPC_MDVALUE(md), GRPC_MDVALUE(ent)) == 0; + r.has_value = grpc_slice_eq(GRPC_MDVALUE(md), GRPC_MDVALUE(ent)); if (r.has_value) return r; } diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index 12f850791d..b002710be3 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -462,13 +462,13 @@ static void on_initial_header(grpc_exec_ctx *exec_ctx, void *tp, gpr_free(value); } - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_STATUS) == 0 && + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_STATUS) && !grpc_mdelem_eq(md, GRPC_MDELEM_GRPC_STATUS_0)) { /* TODO(ctiller): check for a status like " 0" */ s->seen_error = true; } - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_TIMEOUT) == 0) { + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_TIMEOUT)) { gpr_timespec *cached_timeout = grpc_mdelem_get_user_data(md, free_timeout); gpr_timespec timeout; if (cached_timeout == NULL) { @@ -534,7 +534,7 @@ static void on_trailing_header(grpc_exec_ctx *exec_ctx, void *tp, gpr_free(value); } - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_STATUS) == 0 && + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_STATUS) && !grpc_mdelem_eq(md, GRPC_MDELEM_GRPC_STATUS_0)) { /* TODO(ctiller): check for a status like " 0" */ s->seen_error = true; diff --git a/src/core/lib/channel/compress_filter.c b/src/core/lib/channel/compress_filter.c index e04f3dc7db..392d4d7231 100644 --- a/src/core/lib/channel/compress_filter.c +++ b/src/core/lib/channel/compress_filter.c @@ -90,8 +90,8 @@ static grpc_mdelem compression_md_filter(grpc_exec_ctx *exec_ctx, call_data *calld = elem->call_data; channel_data *channeld = elem->channel_data; - if (grpc_slice_cmp(GRPC_MDKEY(md), - GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST) == 0) { + if (grpc_slice_eq(GRPC_MDKEY(md), + GRPC_MDSTR_GRPC_INTERNAL_ENCODING_REQUEST)) { if (!grpc_compression_algorithm_parse(GRPC_MDVALUE(md), &calld->compression_algorithm)) { char *val = grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_ASCII); diff --git a/src/core/lib/channel/http_client_filter.c b/src/core/lib/channel/http_client_filter.c index fea6c9b345..e415877eb8 100644 --- a/src/core/lib/channel/http_client_filter.c +++ b/src/core/lib/channel/http_client_filter.c @@ -99,7 +99,7 @@ static grpc_mdelem client_recv_filter(grpc_exec_ctx *exec_ctx, void *user_data, grpc_call_element *elem = user_data; if (grpc_mdelem_eq(md, GRPC_MDELEM_STATUS_200)) { return GRPC_MDNULL; - } else if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_STATUS) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_STATUS)) { char *message_string; char *val = grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_ASCII); gpr_asprintf(&message_string, "Received http2 header with status: %s", val); @@ -109,7 +109,7 @@ static grpc_mdelem client_recv_filter(grpc_exec_ctx *exec_ctx, void *user_data, grpc_call_element_send_close_with_message(exec_ctx, elem, GRPC_STATUS_CANCELLED, &message); return GRPC_MDNULL; - } else if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_MESSAGE) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_MESSAGE)) { grpc_slice pct_decoded_msg = grpc_permissive_percent_decode_slice(GRPC_MDVALUE(md)); if (grpc_slice_is_equivalent(pct_decoded_msg, GRPC_MDVALUE(md))) { @@ -122,7 +122,7 @@ static grpc_mdelem client_recv_filter(grpc_exec_ctx *exec_ctx, void *user_data, } else if (grpc_mdelem_eq(md, GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC)) { return GRPC_MDNULL; - } else if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_CONTENT_TYPE) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_CONTENT_TYPE)) { if (grpc_slice_buf_start_eq(GRPC_MDVALUE(md), EXPECTED_CONTENT_TYPE, EXPECTED_CONTENT_TYPE_LENGTH) && (GRPC_SLICE_START_PTR(GRPC_MDVALUE(md))[EXPECTED_CONTENT_TYPE_LENGTH] == @@ -187,15 +187,12 @@ static void send_done(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) { static grpc_mdelem client_strip_filter(grpc_exec_ctx *exec_ctx, void *user_data, grpc_mdelem md) { /* eat the things we'd like to set ourselves */ - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_METHOD) == 0) - return GRPC_MDNULL; - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_SCHEME) == 0) - return GRPC_MDNULL; - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_TE) == 0) return GRPC_MDNULL; - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_CONTENT_TYPE) == 0) - return GRPC_MDNULL; - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_USER_AGENT) == 0) + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_METHOD)) return GRPC_MDNULL; + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_SCHEME)) return GRPC_MDNULL; + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_TE)) return GRPC_MDNULL; + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_CONTENT_TYPE)) return GRPC_MDNULL; + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_USER_AGENT)) return GRPC_MDNULL; return md; } diff --git a/src/core/lib/channel/http_server_filter.c b/src/core/lib/channel/http_server_filter.c index 0d11c71d06..bdd1e18837 100644 --- a/src/core/lib/channel/http_server_filter.c +++ b/src/core/lib/channel/http_server_filter.c @@ -87,7 +87,7 @@ typedef struct channel_data { uint8_t unused; } channel_data; static grpc_mdelem server_filter_outgoing_metadata(grpc_exec_ctx *exec_ctx, void *user_data, grpc_mdelem md) { - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_MESSAGE) == 0) { + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_MESSAGE)) { grpc_slice pct_encoded_msg = grpc_percent_encode_slice( GRPC_MDVALUE(md), grpc_compatible_percent_encoding_unreserved_bytes); if (grpc_slice_is_equivalent(pct_encoded_msg, GRPC_MDVALUE(md))) { @@ -126,7 +126,7 @@ static grpc_mdelem server_filter(grpc_exec_ctx *exec_ctx, void *user_data, } else if (grpc_mdelem_eq(md, GRPC_MDELEM_METHOD_GET)) { calld->seen_method = 1; *calld->recv_cacheable_request = true; - } else if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_SCHEME) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_SCHEME)) { calld->seen_scheme = 1; } else if (grpc_mdelem_eq(md, GRPC_MDELEM_TE_TRAILERS)) { calld->seen_te_trailers = 1; @@ -134,7 +134,7 @@ static grpc_mdelem server_filter(grpc_exec_ctx *exec_ctx, void *user_data, /* TODO(klempner): Track that we've seen all the headers we should require */ return GRPC_MDNULL; - } else if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_CONTENT_TYPE) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_CONTENT_TYPE)) { if (grpc_slice_buf_start_eq(GRPC_MDVALUE(md), EXPECTED_CONTENT_TYPE, EXPECTED_CONTENT_TYPE_LENGTH) && (GRPC_SLICE_START_PTR(GRPC_MDVALUE(md))[EXPECTED_CONTENT_TYPE_LENGTH] == @@ -154,9 +154,9 @@ static grpc_mdelem server_filter(grpc_exec_ctx *exec_ctx, void *user_data, gpr_free(val); } return GRPC_MDNULL; - } else if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_TE) == 0 || - grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_METHOD) == 0 || - grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_SCHEME) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_TE) || + grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_METHOD) || + grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_SCHEME)) { char *key = grpc_dump_slice(GRPC_MDKEY(md), GPR_DUMP_ASCII); char *value = grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_ASCII); gpr_log(GPR_ERROR, "Invalid %s: header: '%s'", key, value); @@ -167,24 +167,24 @@ static grpc_mdelem server_filter(grpc_exec_ctx *exec_ctx, void *user_data, gpr_free(value); grpc_call_element_send_cancel(exec_ctx, elem); return GRPC_MDNULL; - } else if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_PATH) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_PATH)) { if (calld->seen_path) { gpr_log(GPR_ERROR, "Received :path twice"); return GRPC_MDNULL; } calld->seen_path = 1; return md; - } else if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_AUTHORITY) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_AUTHORITY)) { calld->seen_authority = 1; return md; - } else if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_HOST) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_HOST)) { /* translate host to :authority since :authority may be omitted */ grpc_mdelem authority = grpc_mdelem_from_slices( exec_ctx, GRPC_MDSTR_AUTHORITY, grpc_slice_ref(GRPC_MDVALUE(md))); calld->seen_authority = 1; return authority; - } else if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_PAYLOAD_BIN) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_PAYLOAD_BIN)) { /* Retrieve the payload from the value of the 'grpc-internal-payload-bin' header field */ calld->seen_payload_bin = 1; diff --git a/src/core/lib/compression/compression.c b/src/core/lib/compression/compression.c index f8777a6f6c..80c1aaed5b 100644 --- a/src/core/lib/compression/compression.c +++ b/src/core/lib/compression/compression.c @@ -47,13 +47,13 @@ int grpc_compression_algorithm_parse(grpc_slice name, * doesn't matter, given that we are comparing against string literals, but * because this way we needn't have "name" nil-terminated (useful for slice * data, for example) */ - if (grpc_slice_cmp(name, GRPC_MDSTR_IDENTITY) == 0) { + if (grpc_slice_eq(name, GRPC_MDSTR_IDENTITY)) { *algorithm = GRPC_COMPRESS_NONE; return 1; - } else if (grpc_slice_cmp(name, GRPC_MDSTR_GZIP) == 0) { + } else if (grpc_slice_eq(name, GRPC_MDSTR_GZIP)) { *algorithm = GRPC_COMPRESS_GZIP; return 1; - } else if (grpc_slice_cmp(name, GRPC_MDSTR_DEFLATE) == 0) { + } else if (grpc_slice_eq(name, GRPC_MDSTR_DEFLATE)) { *algorithm = GRPC_COMPRESS_DEFLATE; return 1; } else { @@ -83,10 +83,10 @@ int grpc_compression_algorithm_name(grpc_compression_algorithm algorithm, grpc_compression_algorithm grpc_compression_algorithm_from_slice( grpc_slice str) { - if (grpc_slice_cmp(str, GRPC_MDSTR_IDENTITY) == 0) return GRPC_COMPRESS_NONE; - if (grpc_slice_cmp(str, GRPC_MDSTR_DEFLATE) == 0) + if (grpc_slice_eq(str, GRPC_MDSTR_IDENTITY)) return GRPC_COMPRESS_NONE; + if (grpc_slice_eq(str, GRPC_MDSTR_DEFLATE)) return GRPC_COMPRESS_DEFLATE; - if (grpc_slice_cmp(str, GRPC_MDSTR_GZIP) == 0) return GRPC_COMPRESS_GZIP; + if (grpc_slice_eq(str, GRPC_MDSTR_GZIP)) return GRPC_COMPRESS_GZIP; return GRPC_COMPRESS_ALGORITHMS_COUNT; } diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 3b63773071..8622a3610c 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -367,7 +367,8 @@ static void ru_slice_unref(grpc_exec_ctx *exec_ctx, void *p) { } static const grpc_slice_refcount_vtable ru_slice_vtable = { - ru_slice_ref, ru_slice_unref, grpc_slice_default_hash_impl}; + ru_slice_ref, ru_slice_unref, grpc_slice_default_eq_impl, + grpc_slice_default_hash_impl}; static grpc_slice ru_slice_create(grpc_resource_user *resource_user, size_t size) { diff --git a/src/core/lib/security/transport/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c index d680c1ef13..13c0277109 100644 --- a/src/core/lib/security/transport/client_auth_filter.c +++ b/src/core/lib/security/transport/client_auth_filter.c @@ -256,13 +256,13 @@ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx, grpc_mdelem md = l->md; /* Pointer comparison is OK for md_elems created from the same context. */ - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_AUTHORITY) == 0) { + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_AUTHORITY)) { if (calld->have_host) { grpc_slice_unref_internal(exec_ctx, calld->host); } calld->host = grpc_slice_ref_internal(GRPC_MDVALUE(md)); calld->have_host = true; - } else if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_PATH) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_PATH)) { if (calld->have_method) { grpc_slice_unref_internal(exec_ctx, calld->method); } diff --git a/src/core/lib/security/transport/server_auth_filter.c b/src/core/lib/security/transport/server_auth_filter.c index 1c4843fc14..1d14cfef53 100644 --- a/src/core/lib/security/transport/server_auth_filter.c +++ b/src/core/lib/security/transport/server_auth_filter.c @@ -90,8 +90,8 @@ static grpc_mdelem remove_consumed_md(grpc_exec_ctx *exec_ctx, void *user_data, size_t i; for (i = 0; i < calld->num_consumed_md; i++) { const grpc_metadata *consumed_md = &calld->consumed_md[i]; - if (grpc_slice_cmp(GRPC_MDKEY(md), consumed_md->key) == 0 && - grpc_slice_cmp(GRPC_MDKEY(md), consumed_md->value) == 0) + if (grpc_slice_eq(GRPC_MDKEY(md), consumed_md->key) && + grpc_slice_eq(GRPC_MDKEY(md), consumed_md->value)) return GRPC_MDNULL; } return md; diff --git a/src/core/lib/slice/slice.c b/src/core/lib/slice/slice.c index 64988bae64..2ae97b3035 100644 --- a/src/core/lib/slice/slice.c +++ b/src/core/lib/slice/slice.c @@ -79,7 +79,8 @@ static void noop_ref(void *unused) {} static void noop_unref(grpc_exec_ctx *exec_ctx, void *unused) {} static const grpc_slice_refcount_vtable noop_refcount_vtable = { - noop_ref, noop_unref, grpc_slice_default_hash_impl}; + noop_ref, noop_unref, grpc_slice_default_eq_impl, + grpc_slice_default_hash_impl}; static grpc_slice_refcount noop_refcount = {&noop_refcount_vtable}; grpc_slice grpc_slice_from_static_buffer(const void *s, size_t len) { @@ -117,7 +118,8 @@ static void new_slice_unref(grpc_exec_ctx *exec_ctx, void *p) { } static const grpc_slice_refcount_vtable new_slice_vtable = { - new_slice_ref, new_slice_unref, grpc_slice_default_hash_impl}; + new_slice_ref, new_slice_unref, grpc_slice_default_eq_impl, + grpc_slice_default_hash_impl}; grpc_slice grpc_slice_new_with_user_data(void *p, size_t len, void (*destroy)(void *), @@ -164,7 +166,8 @@ static void new_with_len_unref(grpc_exec_ctx *exec_ctx, void *p) { } static const grpc_slice_refcount_vtable new_with_len_vtable = { - new_with_len_ref, new_with_len_unref, grpc_slice_default_hash_impl}; + new_with_len_ref, new_with_len_unref, grpc_slice_default_eq_impl, + grpc_slice_default_hash_impl}; grpc_slice grpc_slice_new_with_len(void *p, size_t len, void (*destroy)(void *, size_t)) { @@ -211,7 +214,8 @@ static void malloc_unref(grpc_exec_ctx *exec_ctx, void *p) { } static const grpc_slice_refcount_vtable malloc_vtable = { - malloc_ref, malloc_unref, grpc_slice_default_hash_impl}; + malloc_ref, malloc_unref, grpc_slice_default_eq_impl, + grpc_slice_default_hash_impl}; grpc_slice grpc_slice_malloc(size_t length) { grpc_slice slice; @@ -363,11 +367,20 @@ grpc_slice grpc_slice_split_head(grpc_slice *source, size_t split) { return head; } -int grpc_slice_cmp(grpc_slice a, grpc_slice b) { - if (GRPC_SLICE_START_PTR(a) == GRPC_SLICE_START_PTR(b) && - GRPC_SLICE_LENGTH(a) == GRPC_SLICE_LENGTH(b)) { - return 0; +int grpc_slice_default_eq_impl(grpc_slice a, grpc_slice b) { + return GRPC_SLICE_LENGTH(a) == GRPC_SLICE_LENGTH(b) && + 0 == memcmp(GRPC_SLICE_START_PTR(a), GRPC_SLICE_START_PTR(b), + GRPC_SLICE_LENGTH(a)); +} + +int grpc_slice_eq(grpc_slice a, grpc_slice b) { + if (a.refcount && b.refcount && a.refcount->vtable == b.refcount->vtable) { + return a.refcount->vtable->eq(a, b); } + return grpc_slice_default_eq_impl(a, b); +} + +int grpc_slice_cmp(grpc_slice a, grpc_slice b) { int d = (int)(GRPC_SLICE_LENGTH(a) - GRPC_SLICE_LENGTH(b)); if (d != 0) return d; return memcmp(GRPC_SLICE_START_PTR(a), GRPC_SLICE_START_PTR(b), @@ -383,7 +396,7 @@ int grpc_slice_str_cmp(grpc_slice a, const char *b) { int grpc_slice_is_equivalent(grpc_slice a, grpc_slice b) { if (a.refcount == NULL || b.refcount == NULL) { - return grpc_slice_cmp(a, b) == 0; + return grpc_slice_eq(a, b); } return a.data.refcounted.length == b.data.refcounted.length && a.data.refcounted.bytes == b.data.refcounted.bytes; diff --git a/src/core/lib/slice/slice_intern.c b/src/core/lib/slice/slice_intern.c index 2d93284926..3cae8a98f3 100644 --- a/src/core/lib/slice/slice_intern.c +++ b/src/core/lib/slice/slice_intern.c @@ -110,17 +110,30 @@ static void interned_slice_unref(grpc_exec_ctx *exec_ctx, void *p) { } } -static uint32_t interned_slice_hash(void *p, grpc_slice slice) { - interned_slice_refcount *s = p; +static uint32_t interned_slice_hash(grpc_slice slice) { + interned_slice_refcount *s = (interned_slice_refcount *)slice.refcount; if (slice.data.refcounted.bytes == (uint8_t *)(s + 1) && slice.data.refcounted.length == s->length) { return s->hash; } - return grpc_slice_default_hash_impl(p, slice); + return grpc_slice_default_hash_impl(slice); +} + +static int interned_slice_eq(grpc_slice a, grpc_slice b) { + interned_slice_refcount *sa = (interned_slice_refcount *)a.refcount; + interned_slice_refcount *sb = (interned_slice_refcount *)b.refcount; + if (a.data.refcounted.bytes == (uint8_t *)(sa + 1) && + b.data.refcounted.bytes == (uint8_t *)(sb + 1)) { + return a.data.refcounted.length == b.data.refcounted.length && + a.data.refcounted.bytes == b.data.refcounted.bytes; + } else { + return grpc_slice_default_eq_impl(a, b); + } } static const grpc_slice_refcount_vtable interned_slice_vtable = { - interned_slice_ref, interned_slice_unref, interned_slice_hash}; + interned_slice_ref, interned_slice_unref, interned_slice_eq, + interned_slice_hash}; static void grow_shard(slice_shard *shard) { size_t capacity = shard->capacity * 2; @@ -157,22 +170,31 @@ static grpc_slice materialize(interned_slice_refcount *s) { return slice; } -uint32_t grpc_slice_default_hash_impl(void *unused_refcnt, grpc_slice s) { +uint32_t grpc_slice_default_hash_impl(grpc_slice s) { return gpr_murmur_hash3(GRPC_SLICE_START_PTR(s), GRPC_SLICE_LENGTH(s), g_hash_seed); } -uint32_t grpc_static_slice_hash(void *unused_refcnt, grpc_slice s) { +uint32_t grpc_static_slice_hash(grpc_slice s) { int id = grpc_static_metadata_index(s); if (id == -1) { - return grpc_slice_default_hash_impl(unused_refcnt, s); + return grpc_slice_default_hash_impl(s); } return static_metadata_hash_values[id]; } +int grpc_static_slice_eq(grpc_slice a, grpc_slice b) { + int id_a = grpc_static_metadata_index(a); + int id_b = grpc_static_metadata_index(b); + if (id_a == -1 || id_b == -1) { + return grpc_slice_default_eq_impl(a, b); + } + return id_a == id_b; +} + uint32_t grpc_slice_hash(grpc_slice s) { - return s.refcount == NULL ? grpc_slice_default_hash_impl(NULL, s) - : s.refcount->vtable->hash(s.refcount, s); + return s.refcount == NULL ? grpc_slice_default_hash_impl(s) + : s.refcount->vtable->hash(s); } void grpc_slice_static_intern(grpc_slice *slice) { @@ -185,7 +207,7 @@ void grpc_slice_static_intern(grpc_slice *slice) { static_metadata_hash_ent ent = static_metadata_hash[(hash + i) % GPR_ARRAY_SIZE(static_metadata_hash)]; if (ent.hash == hash && ent.idx < GRPC_STATIC_MDSTR_COUNT && - 0 == grpc_slice_cmp(grpc_static_slice_table[ent.idx], *slice)) { + grpc_slice_eq(grpc_static_slice_table[ent.idx], *slice)) { grpc_slice_unref(*slice); *slice = grpc_static_slice_table[ent.idx]; return; @@ -208,7 +230,7 @@ grpc_slice grpc_slice_intern(grpc_slice slice) { static_metadata_hash_ent ent = static_metadata_hash[(hash + i) % GPR_ARRAY_SIZE(static_metadata_hash)]; if (ent.hash == hash && ent.idx < GRPC_STATIC_MDSTR_COUNT && - 0 == grpc_slice_cmp(grpc_static_slice_table[ent.idx], slice)) { + grpc_slice_eq(grpc_static_slice_table[ent.idx], slice)) { return grpc_static_slice_table[ent.idx]; } } @@ -221,7 +243,7 @@ grpc_slice grpc_slice_intern(grpc_slice slice) { /* search for an existing string */ size_t idx = TABLE_IDX(hash, shard->capacity); for (s = shard->strs[idx]; s; s = s->bucket_next) { - if (s->hash == hash && grpc_slice_cmp(slice, materialize(s)) == 0) { + if (s->hash == hash && grpc_slice_eq(slice, materialize(s))) { if (gpr_atm_no_barrier_fetch_add(&s->refcnt, 1) == 0) { /* If we get here, we've added a ref to something that was about to * die - drop it immediately. @@ -283,7 +305,7 @@ void grpc_slice_intern_init(void) { max_static_metadata_hash_probe = 0; for (size_t i = 0; i < GRPC_STATIC_MDSTR_COUNT; i++) { static_metadata_hash_values[i] = - grpc_slice_default_hash_impl(NULL, grpc_static_slice_table[i]); + grpc_slice_default_hash_impl(grpc_static_slice_table[i]); for (size_t j = 0; j < GPR_ARRAY_SIZE(static_metadata_hash); j++) { size_t slot = (static_metadata_hash_values[i] + j) % GPR_ARRAY_SIZE(static_metadata_hash); diff --git a/src/core/lib/slice/slice_internal.h b/src/core/lib/slice/slice_internal.h index 211d5f06be..cb85d57213 100644 --- a/src/core/lib/slice/slice_internal.h +++ b/src/core/lib/slice/slice_internal.h @@ -55,6 +55,7 @@ void grpc_test_only_set_slice_hash_seed(uint32_t key); // if slice matches a static slice, consume it and replace it with the static // slice, otherwise do nothing: this is a fast interning for well known strings void grpc_slice_static_intern(grpc_slice *slice); -uint32_t grpc_static_slice_hash(void *refcnt, grpc_slice s); +uint32_t grpc_static_slice_hash(grpc_slice s); +int grpc_static_slice_eq(grpc_slice a, grpc_slice b); #endif /* GRPC_CORE_LIB_SLICE_SLICE_INTERNAL_H */ diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 0ca97337e2..af53a5b246 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -252,8 +252,8 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, MAX_SEND_EXTRA_METADATA_COUNT); for (i = 0; i < args->add_initial_metadata_count; i++) { call->send_extra_metadata[i].md = args->add_initial_metadata[i]; - if (grpc_slice_cmp(GRPC_MDKEY(args->add_initial_metadata[i]), - GRPC_MDSTR_PATH) == 0) { + if (grpc_slice_eq(GRPC_MDKEY(args->add_initial_metadata[i]), + GRPC_MDSTR_PATH)) { path = grpc_slice_ref_internal( GRPC_MDVALUE(args->add_initial_metadata[i])); } @@ -916,12 +916,12 @@ static grpc_compression_algorithm decode_compression(grpc_mdelem md) { static grpc_mdelem recv_common_filter(grpc_exec_ctx *exec_ctx, grpc_call *call, grpc_mdelem elem) { - if (grpc_slice_cmp(GRPC_MDKEY(elem), GRPC_MDSTR_GRPC_STATUS) == 0) { + if (grpc_slice_eq(GRPC_MDKEY(elem), GRPC_MDSTR_GRPC_STATUS)) { GPR_TIMER_BEGIN("status", 0); set_status_code(call, STATUS_FROM_WIRE, decode_status(elem)); GPR_TIMER_END("status", 0); return GRPC_MDNULL; - } else if (grpc_slice_cmp(GRPC_MDKEY(elem), GRPC_MDSTR_GRPC_MESSAGE) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(elem), GRPC_MDSTR_GRPC_MESSAGE)) { GPR_TIMER_BEGIN("status-details", 0); set_status_details(exec_ctx, call, STATUS_FROM_WIRE, grpc_slice_ref_internal(GRPC_MDVALUE(elem))); @@ -955,13 +955,12 @@ static grpc_mdelem recv_initial_filter(grpc_exec_ctx *exec_ctx, void *args, elem = recv_common_filter(exec_ctx, call, elem); if (GRPC_MDISNULL(elem)) { return GRPC_MDNULL; - } else if (grpc_slice_cmp(GRPC_MDKEY(elem), GRPC_MDSTR_GRPC_ENCODING) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(elem), GRPC_MDSTR_GRPC_ENCODING)) { GPR_TIMER_BEGIN("incoming_compression_algorithm", 0); set_incoming_compression_algorithm(call, decode_compression(elem)); GPR_TIMER_END("incoming_compression_algorithm", 0); return GRPC_MDNULL; - } else if (grpc_slice_cmp(GRPC_MDKEY(elem), - GRPC_MDSTR_GRPC_ACCEPT_ENCODING) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(elem), GRPC_MDSTR_GRPC_ACCEPT_ENCODING)) { GPR_TIMER_BEGIN("encodings_accepted_by_peer", 0); set_encodings_accepted_by_peer(exec_ctx, call, elem); GPR_TIMER_END("encodings_accepted_by_peer", 0); diff --git a/src/core/lib/surface/channel.c b/src/core/lib/surface/channel.c index 501545b5c0..6f2a88ae68 100644 --- a/src/core/lib/surface/channel.c +++ b/src/core/lib/surface/channel.c @@ -125,7 +125,8 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target, } channel->default_authority = grpc_mdelem_from_slices( exec_ctx, GRPC_MDSTR_AUTHORITY, - grpc_slice_from_copied_string(args->args[i].value.string)); + grpc_slice_intern( + grpc_slice_from_copied_string(args->args[i].value.string))); } } else if (0 == strcmp(args->args[i].key, GRPC_SSL_TARGET_NAME_OVERRIDE_ARG)) { @@ -141,7 +142,8 @@ grpc_channel *grpc_channel_create(grpc_exec_ctx *exec_ctx, const char *target, } else { channel->default_authority = grpc_mdelem_from_slices( exec_ctx, GRPC_MDSTR_AUTHORITY, - grpc_slice_from_copied_string(args->args[i].value.string)); + grpc_slice_intern( + grpc_slice_from_copied_string(args->args[i].value.string))); } } } else if (0 == strcmp(args->args[i].key, diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index b21e1d8113..1c29873a65 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -624,8 +624,8 @@ static void start_new_rpc(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { chand->registered_method_slots]; if (!rm) break; if (!rm->has_host) continue; - if (grpc_slice_cmp(rm->host, calld->host) != 0) continue; - if (grpc_slice_cmp(rm->method, calld->path) != 0) continue; + if (!grpc_slice_eq(rm->host, calld->host)) continue; + if (!grpc_slice_eq(rm->method, calld->path)) continue; if ((rm->flags & GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) && !calld->recv_idempotent_request) { continue; @@ -642,7 +642,7 @@ static void start_new_rpc(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { chand->registered_method_slots]; if (!rm) break; if (rm->has_host) continue; - if (grpc_slice_cmp(rm->method, calld->path) != 0) continue; + if (!grpc_slice_eq(rm->method, calld->path)) continue; if ((rm->flags & GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) && !calld->recv_idempotent_request) { continue; @@ -739,13 +739,13 @@ static grpc_mdelem server_filter(grpc_exec_ctx *exec_ctx, void *user_data, grpc_mdelem md) { grpc_call_element *elem = user_data; call_data *calld = elem->call_data; - if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_PATH) == 0) { + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_PATH)) { if (!calld->path_set) { calld->path = grpc_slice_ref(GRPC_MDVALUE(md)); calld->path_set = true; } return GRPC_MDNULL; - } else if (grpc_slice_cmp(GRPC_MDKEY(md), GRPC_MDSTR_AUTHORITY) == 0) { + } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_AUTHORITY)) { if (!calld->host_set) { calld->host = grpc_slice_ref(GRPC_MDVALUE(md)); calld->host_set = true; diff --git a/src/core/lib/transport/metadata.c b/src/core/lib/transport/metadata.c index 2082d1bac2..f9f5ac9dd2 100644 --- a/src/core/lib/transport/metadata.c +++ b/src/core/lib/transport/metadata.c @@ -279,8 +279,7 @@ grpc_mdelem grpc_mdelem_create( idx = TABLE_IDX(hash, shard->capacity); /* search for an existing pair */ for (md = shard->elems[idx]; md; md = md->bucket_next) { - if (grpc_slice_cmp(key, md->key) == 0 && - grpc_slice_cmp(value, md->value) == 0) { + if (grpc_slice_eq(key, md->key) && grpc_slice_eq(value, md->value)) { REF_MD_LOCKED(shard, md); gpr_mu_unlock(&shard->mu); GPR_TIMER_END("grpc_mdelem_from_metadata_strings", 0); @@ -496,7 +495,8 @@ void *grpc_mdelem_set_user_data(grpc_mdelem md, void (*destroy_func)(void *), bool grpc_mdelem_eq(grpc_mdelem a, grpc_mdelem b) { if (a.payload == b.payload) return true; + if (GRPC_MDELEM_IS_INTERNED(a) && GRPC_MDELEM_IS_INTERNED(b)) return false; if (GRPC_MDISNULL(a) || GRPC_MDISNULL(b)) return false; - return 0 == grpc_slice_cmp(GRPC_MDKEY(a), GRPC_MDKEY(b)) && - 0 == grpc_slice_cmp(GRPC_MDVALUE(a), GRPC_MDVALUE(b)); + return grpc_slice_eq(GRPC_MDKEY(a), GRPC_MDKEY(b)) && + grpc_slice_eq(GRPC_MDVALUE(a), GRPC_MDVALUE(b)); } diff --git a/src/core/lib/transport/metadata.h b/src/core/lib/transport/metadata.h index 6d0afbe789..f4ba86c854 100644 --- a/src/core/lib/transport/metadata.h +++ b/src/core/lib/transport/metadata.h @@ -86,17 +86,21 @@ typedef struct grpc_mdelem_data { /* there is a private part to this in metadata.c */ } grpc_mdelem_data; +/* GRPC_MDELEM_STORAGE_* enum values that can be treated as interned always have + this bit set in their integer value */ +#define GRPC_MDELEM_STORAGE_INTERNED_BIT 1 + typedef enum { /* memory pointed to by grpc_mdelem::payload is owned by an external system */ GRPC_MDELEM_STORAGE_EXTERNAL = 0, /* memory pointed to by grpc_mdelem::payload is interned by the metadata system */ - GRPC_MDELEM_STORAGE_INTERNED = 1, + GRPC_MDELEM_STORAGE_INTERNED = GRPC_MDELEM_STORAGE_INTERNED_BIT, /* memory pointed to by grpc_mdelem::payload is allocated by the metadata system */ GRPC_MDELEM_STORAGE_ALLOCATED = 2, /* memory is in the static metadata table */ - GRPC_MDELEM_STORAGE_STATIC = 3, + GRPC_MDELEM_STORAGE_STATIC = 2 | GRPC_MDELEM_STORAGE_INTERNED_BIT, } grpc_mdelem_data_storage; struct grpc_mdelem { @@ -111,6 +115,9 @@ struct grpc_mdelem { ((grpc_mdelem_data_storage)((md).payload & (uintptr_t)3)) #define GRPC_MAKE_MDELEM(data, storage) \ ((grpc_mdelem){((uintptr_t)(data)) | ((uintptr_t)storage)}) +#define GRPC_MDELEM_IS_INTERNED(md) \ + ((grpc_mdelem_data_storage)((md).payload & \ + (uintptr_t)GRPC_MDELEM_STORAGE_INTERNED_BIT)) /* Unrefs the slices. */ grpc_mdelem grpc_mdelem_from_slices(grpc_exec_ctx *exec_ctx, grpc_slice key, diff --git a/src/core/lib/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c index 575f442a9b..374a5fd615 100644 --- a/src/core/lib/transport/static_metadata.c +++ b/src/core/lib/transport/static_metadata.c @@ -117,7 +117,7 @@ static uint8_t g_raw_bytes[] = { static void static_ref(void *unused) {} static void static_unref(grpc_exec_ctx *exec_ctx, void *unused) {} static const grpc_slice_refcount_vtable static_vtable = { - static_ref, static_unref, grpc_static_slice_hash}; + static_ref, static_unref, grpc_static_slice_eq, grpc_static_slice_hash}; static grpc_slice_refcount g_refcnt = {&static_vtable}; bool grpc_is_static_metadata_string(grpc_slice slice) { |