diff options
Diffstat (limited to 'src/core/ext/transport/chttp2')
39 files changed, 2835 insertions, 3658 deletions
diff --git a/src/core/ext/transport/chttp2/alpn/alpn.c b/src/core/ext/transport/chttp2/alpn/alpn.c index 48b0217265..55710dc5ae 100644 --- a/src/core/ext/transport/chttp2/alpn/alpn.c +++ b/src/core/ext/transport/chttp2/alpn/alpn.c @@ -36,7 +36,7 @@ #include <grpc/support/useful.h> /* in order of preference */ -static const char *const supported_versions[] = {"h2"}; +static const char *const supported_versions[] = {"grpc-exp", "h2"}; int grpc_chttp2_is_alpn_version_supported(const char *version, size_t size) { size_t i; diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create.c b/src/core/ext/transport/chttp2/client/insecure/channel_create.c index c2b59569fd..8e03fd82c1 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create.c @@ -36,13 +36,13 @@ #include <stdlib.h> #include <string.h> +#include <grpc/slice.h> +#include <grpc/slice_buffer.h> #include <grpc/support/alloc.h> -#include <grpc/support/slice.h> -#include <grpc/support/slice_buffer.h> -#include "src/core/ext/client_config/client_channel.h" -#include "src/core/ext/client_config/http_connect_handshaker.h" -#include "src/core/ext/client_config/resolver_registry.h" +#include "src/core/ext/client_channel/client_channel.h" +#include "src/core/ext/client_channel/http_connect_handshaker.h" +#include "src/core/ext/client_channel/resolver_registry.h" #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/compress_filter.h" @@ -52,6 +52,10 @@ #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" +// +// connector +// + typedef struct { grpc_connector base; gpr_refcount refs; @@ -60,7 +64,7 @@ typedef struct { grpc_connect_in_args args; grpc_connect_out_args *result; grpc_closure initial_string_sent; - gpr_slice_buffer initial_string_buffer; + grpc_slice_buffer initial_string_buffer; grpc_endpoint *tcp; @@ -90,7 +94,7 @@ static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg, static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint, grpc_channel_args *args, - gpr_slice_buffer *read_buffer, void *user_data, + grpc_slice_buffer *read_buffer, void *user_data, grpc_error *error) { connector *c = user_data; if (error != GRPC_ERROR_NONE) { @@ -113,12 +117,12 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { connector *c = arg; grpc_endpoint *tcp = c->tcp; if (tcp != NULL) { - if (!GPR_SLICE_IS_EMPTY(c->args.initial_connect_string)) { + if (!GRPC_SLICE_IS_EMPTY(c->args.initial_connect_string)) { grpc_closure_init(&c->initial_string_sent, on_initial_connect_string_sent, c); - gpr_slice_buffer_init(&c->initial_string_buffer); - gpr_slice_buffer_add(&c->initial_string_buffer, - c->args.initial_connect_string); + grpc_slice_buffer_init(&c->initial_string_buffer); + grpc_slice_buffer_add(&c->initial_string_buffer, + c->args.initial_connect_string); connector_ref(arg); grpc_endpoint_write(exec_ctx, tcp, &c->initial_string_buffer, &c->initial_string_sent); @@ -150,42 +154,27 @@ static void connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *con, c->tcp = NULL; grpc_closure_init(&c->connected, connected, c); grpc_tcp_client_connect(exec_ctx, &c->connected, &c->tcp, - args->interested_parties, args->addr, args->addr_len, - args->deadline); + args->interested_parties, args->channel_args, + args->addr, args->deadline); } static const grpc_connector_vtable connector_vtable = { connector_ref, connector_unref, connector_shutdown, connector_connect}; -typedef struct { - grpc_client_channel_factory base; - gpr_refcount refs; - grpc_channel_args *merge_args; -} client_channel_factory; +// +// client_channel_factory +// static void client_channel_factory_ref( - grpc_client_channel_factory *cc_factory) { - client_channel_factory *f = (client_channel_factory *)cc_factory; - gpr_ref(&f->refs); -} + grpc_client_channel_factory *cc_factory) {} static void client_channel_factory_unref( - grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory) { - client_channel_factory *f = (client_channel_factory *)cc_factory; - if (gpr_unref(&f->refs)) { - grpc_channel_args_destroy(f->merge_args); - gpr_free(f); - } -} + grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory) {} static grpc_subchannel *client_channel_factory_create_subchannel( grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory, - grpc_subchannel_args *args) { - client_channel_factory *f = (client_channel_factory *)cc_factory; + const grpc_subchannel_args *args) { connector *c = gpr_malloc(sizeof(*c)); - grpc_channel_args *final_args = - grpc_channel_args_merge(args->args, f->merge_args); - grpc_subchannel *s; memset(c, 0, sizeof(*c)); c->base.vtable = &connector_vtable; gpr_ref_init(&c->refs, 1); @@ -197,23 +186,18 @@ static grpc_subchannel *client_channel_factory_create_subchannel( grpc_http_connect_handshaker_create(proxy_name, args->server_name)); gpr_free(proxy_name); } - args->args = final_args; - s = grpc_subchannel_create(exec_ctx, &c->base, args); + grpc_subchannel *s = grpc_subchannel_create(exec_ctx, &c->base, args); grpc_connector_unref(exec_ctx, &c->base); - grpc_channel_args_destroy(final_args); return s; } static grpc_channel *client_channel_factory_create_channel( grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory, const char *target, grpc_client_channel_type type, - grpc_channel_args *args) { - client_channel_factory *f = (client_channel_factory *)cc_factory; - grpc_channel_args *final_args = grpc_channel_args_merge(args, f->merge_args); - grpc_channel *channel = grpc_channel_create(exec_ctx, target, final_args, - GRPC_CLIENT_CHANNEL, NULL); - grpc_channel_args_destroy(final_args); - grpc_resolver *resolver = grpc_resolver_create(target); + const grpc_channel_args *args) { + grpc_channel *channel = + grpc_channel_create(exec_ctx, target, args, GRPC_CLIENT_CHANNEL, NULL); + grpc_resolver *resolver = grpc_resolver_create(target, args); if (!resolver) { GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel, "client_channel_factory_create_channel"); @@ -221,7 +205,7 @@ static grpc_channel *client_channel_factory_create_channel( } grpc_client_channel_finish_initialization( - exec_ctx, grpc_channel_get_channel_stack(channel), resolver, &f->base); + exec_ctx, grpc_channel_get_channel_stack(channel), resolver, cc_factory); GRPC_RESOLVER_UNREF(exec_ctx, resolver, "create_channel"); return channel; @@ -232,6 +216,9 @@ static const grpc_client_channel_factory_vtable client_channel_factory_vtable = client_channel_factory_create_subchannel, client_channel_factory_create_channel}; +static grpc_client_channel_factory client_channel_factory = { + &client_channel_factory_vtable}; + /* Create a client channel: Asynchronously: - resolve target - connect to it (trying alternatives as presented) @@ -245,16 +232,12 @@ grpc_channel *grpc_insecure_channel_create(const char *target, (target, args, reserved)); GPR_ASSERT(!reserved); - client_channel_factory *f = gpr_malloc(sizeof(*f)); - memset(f, 0, sizeof(*f)); - f->base.vtable = &client_channel_factory_vtable; - gpr_ref_init(&f->refs, 1); - f->merge_args = grpc_channel_args_copy(args); - + grpc_client_channel_factory *factory = + (grpc_client_channel_factory *)&client_channel_factory; grpc_channel *channel = client_channel_factory_create_channel( - &exec_ctx, &f->base, target, GRPC_CLIENT_CHANNEL_TYPE_REGULAR, NULL); + &exec_ctx, factory, target, GRPC_CLIENT_CHANNEL_TYPE_REGULAR, args); - grpc_client_channel_factory_unref(&exec_ctx, &f->base); + grpc_client_channel_factory_unref(&exec_ctx, factory); grpc_exec_ctx_finish(&exec_ctx); return channel != NULL ? channel : grpc_lame_client_channel_create( diff --git a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c index b2c5e5b088..1e5b1c22e3 100644 --- a/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c +++ b/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c @@ -44,6 +44,7 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/iomgr/tcp_client_posix.h" #include "src/core/lib/iomgr/tcp_posix.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" @@ -65,9 +66,8 @@ grpc_channel *grpc_insecure_channel_create_from_fd( int flags = fcntl(fd, F_GETFL, 0); GPR_ASSERT(fcntl(fd, F_SETFL, flags | O_NONBLOCK) == 0); - grpc_endpoint *client = - grpc_tcp_create(grpc_fd_create(fd, "client"), - GRPC_TCP_DEFAULT_READ_SLICE_SIZE, "fd-client"); + grpc_endpoint *client = grpc_tcp_client_create_from_fd( + &exec_ctx, grpc_fd_create(fd, "client"), args, "fd-client"); grpc_transport *transport = grpc_create_chttp2_transport(&exec_ctx, final_args, client, 1); diff --git a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c index 31c54ff74c..04c88a2d36 100644 --- a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c +++ b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c @@ -36,13 +36,13 @@ #include <stdlib.h> #include <string.h> +#include <grpc/slice.h> +#include <grpc/slice_buffer.h> #include <grpc/support/alloc.h> -#include <grpc/support/slice.h> -#include <grpc/support/slice_buffer.h> -#include "src/core/ext/client_config/client_channel.h" -#include "src/core/ext/client_config/http_connect_handshaker.h" -#include "src/core/ext/client_config/resolver_registry.h" +#include "src/core/ext/client_channel/client_channel.h" +#include "src/core/ext/client_channel/http_connect_handshaker.h" +#include "src/core/ext/client_channel/resolver_registry.h" #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/handshaker.h" @@ -54,6 +54,10 @@ #include "src/core/lib/surface/channel.h" #include "src/core/lib/tsi/transport_security_interface.h" +// +// connector +// + typedef struct { grpc_connector base; gpr_refcount refs; @@ -64,7 +68,7 @@ typedef struct { grpc_connect_in_args args; grpc_connect_out_args *result; grpc_closure initial_string_sent; - gpr_slice_buffer initial_string_buffer; + grpc_slice_buffer initial_string_buffer; gpr_mu mu; grpc_endpoint *connecting_endpoint; @@ -127,7 +131,7 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint, grpc_channel_args *args, - gpr_slice_buffer *read_buffer, void *user_data, + grpc_slice_buffer *read_buffer, void *user_data, grpc_error *error) { connector *c = user_data; c->tmp_args = args; @@ -162,12 +166,12 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { GPR_ASSERT(c->connecting_endpoint == NULL); c->connecting_endpoint = tcp; gpr_mu_unlock(&c->mu); - if (!GPR_SLICE_IS_EMPTY(c->args.initial_connect_string)) { + if (!GRPC_SLICE_IS_EMPTY(c->args.initial_connect_string)) { grpc_closure_init(&c->initial_string_sent, on_initial_connect_string_sent, c); - gpr_slice_buffer_init(&c->initial_string_buffer); - gpr_slice_buffer_add(&c->initial_string_buffer, - c->args.initial_connect_string); + grpc_slice_buffer_init(&c->initial_string_buffer); + grpc_slice_buffer_add(&c->initial_string_buffer, + c->args.initial_connect_string); grpc_endpoint_write(exec_ctx, tcp, &c->initial_string_buffer, &c->initial_string_sent); } else { @@ -210,16 +214,19 @@ static void connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *con, grpc_closure_init(&c->connected_closure, connected, c); grpc_tcp_client_connect( exec_ctx, &c->connected_closure, &c->newly_connecting_endpoint, - args->interested_parties, args->addr, args->addr_len, args->deadline); + args->interested_parties, args->channel_args, args->addr, args->deadline); } static const grpc_connector_vtable connector_vtable = { connector_ref, connector_unref, connector_shutdown, connector_connect}; +// +// client_channel_factory +// + typedef struct { grpc_client_channel_factory base; gpr_refcount refs; - grpc_channel_args *merge_args; grpc_channel_security_connector *security_connector; } client_channel_factory; @@ -235,19 +242,15 @@ static void client_channel_factory_unref( if (gpr_unref(&f->refs)) { GRPC_SECURITY_CONNECTOR_UNREF(&f->security_connector->base, "client_channel_factory"); - grpc_channel_args_destroy(f->merge_args); gpr_free(f); } } static grpc_subchannel *client_channel_factory_create_subchannel( grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory, - grpc_subchannel_args *args) { + const grpc_subchannel_args *args) { client_channel_factory *f = (client_channel_factory *)cc_factory; connector *c = gpr_malloc(sizeof(*c)); - grpc_channel_args *final_args = - grpc_channel_args_merge(args->args, f->merge_args); - grpc_subchannel *s; memset(c, 0, sizeof(*c)); c->base.vtable = &connector_vtable; c->security_connector = f->security_connector; @@ -261,25 +264,19 @@ static grpc_subchannel *client_channel_factory_create_subchannel( } gpr_mu_init(&c->mu); gpr_ref_init(&c->refs, 1); - args->args = final_args; - s = grpc_subchannel_create(exec_ctx, &c->base, args); + grpc_subchannel *s = grpc_subchannel_create(exec_ctx, &c->base, args); grpc_connector_unref(exec_ctx, &c->base); - grpc_channel_args_destroy(final_args); return s; } static grpc_channel *client_channel_factory_create_channel( grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory, const char *target, grpc_client_channel_type type, - grpc_channel_args *args) { + const grpc_channel_args *args) { client_channel_factory *f = (client_channel_factory *)cc_factory; - - grpc_channel_args *final_args = grpc_channel_args_merge(args, f->merge_args); - grpc_channel *channel = grpc_channel_create(exec_ctx, target, final_args, - GRPC_CLIENT_CHANNEL, NULL); - grpc_channel_args_destroy(final_args); - - grpc_resolver *resolver = grpc_resolver_create(target); + grpc_channel *channel = + grpc_channel_create(exec_ctx, target, args, GRPC_CLIENT_CHANNEL, NULL); + grpc_resolver *resolver = grpc_resolver_create(target, args); if (resolver != NULL) { grpc_client_channel_finish_initialization( exec_ctx, grpc_channel_get_channel_stack(channel), resolver, &f->base); @@ -289,9 +286,6 @@ static grpc_channel *client_channel_factory_create_channel( "client_channel_factory_create_channel"); channel = NULL; } - - GRPC_SECURITY_CONNECTOR_UNREF(&f->security_connector->base, - "client_channel_factory_create_channel"); return channel; } @@ -308,19 +302,13 @@ grpc_channel *grpc_secure_channel_create(grpc_channel_credentials *creds, const char *target, const grpc_channel_args *args, void *reserved) { - grpc_arg connector_arg; - grpc_channel_args *args_copy; - grpc_channel_args *new_args_from_connector; - grpc_channel_security_connector *security_connector; - client_channel_factory *f; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - GRPC_API_TRACE( "grpc_secure_channel_create(creds=%p, target=%s, args=%p, " "reserved=%p)", 4, (creds, target, args, reserved)); GPR_ASSERT(reserved == NULL); - + // Make sure security connector does not already exist in args. if (grpc_find_security_connector_in_args(args) != NULL) { gpr_log(GPR_ERROR, "Cannot set security context in channel args."); grpc_exec_ctx_finish(&exec_ctx); @@ -328,7 +316,9 @@ grpc_channel *grpc_secure_channel_create(grpc_channel_credentials *creds, target, GRPC_STATUS_INTERNAL, "Security connector exists in channel args."); } - + // Create security connector and construct new channel args. + grpc_channel_security_connector *security_connector; + grpc_channel_args *new_args_from_connector; if (grpc_channel_credentials_create_security_connector( creds, target, args, &security_connector, &new_args_from_connector) != GRPC_SECURITY_OK) { @@ -336,32 +326,30 @@ grpc_channel *grpc_secure_channel_create(grpc_channel_credentials *creds, return grpc_lame_client_channel_create( target, GRPC_STATUS_INTERNAL, "Failed to create security connector."); } - - connector_arg = grpc_security_connector_to_arg(&security_connector->base); - args_copy = grpc_channel_args_copy_and_add( + grpc_arg connector_arg = + grpc_security_connector_to_arg(&security_connector->base); + grpc_channel_args *new_args = grpc_channel_args_copy_and_add( new_args_from_connector != NULL ? new_args_from_connector : args, &connector_arg, 1); - - f = gpr_malloc(sizeof(*f)); - memset(f, 0, sizeof(*f)); - f->base.vtable = &client_channel_factory_vtable; - gpr_ref_init(&f->refs, 1); - - f->merge_args = grpc_channel_args_copy(args_copy); - grpc_channel_args_destroy(args_copy); if (new_args_from_connector != NULL) { grpc_channel_args_destroy(new_args_from_connector); } - + // Create client channel factory. + client_channel_factory *f = gpr_malloc(sizeof(*f)); + memset(f, 0, sizeof(*f)); + f->base.vtable = &client_channel_factory_vtable; + gpr_ref_init(&f->refs, 1); GRPC_SECURITY_CONNECTOR_REF(&security_connector->base, "grpc_secure_channel_create"); f->security_connector = security_connector; - + // Create channel. grpc_channel *channel = client_channel_factory_create_channel( - &exec_ctx, &f->base, target, GRPC_CLIENT_CHANNEL_TYPE_REGULAR, NULL); - + &exec_ctx, &f->base, target, GRPC_CLIENT_CHANNEL_TYPE_REGULAR, new_args); + // Clean up. + GRPC_SECURITY_CONNECTOR_UNREF(&f->security_connector->base, + "secure_client_channel_factory_create_channel"); + grpc_channel_args_destroy(new_args); grpc_client_channel_factory_unref(&exec_ctx, &f->base); grpc_exec_ctx_finish(&exec_ctx); - return channel; /* may be NULL */ } diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c index f0e07429fa..c18d618f96 100644 --- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c +++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c @@ -56,7 +56,7 @@ typedef struct server_connect_state { static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint, grpc_channel_args *args, - gpr_slice_buffer *read_buffer, void *user_data, + grpc_slice_buffer *read_buffer, void *user_data, grpc_error *error) { server_connect_state *state = user_data; if (error != GRPC_ERROR_NONE) { @@ -139,8 +139,8 @@ int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) { goto error; } - err = - grpc_tcp_server_create(NULL, grpc_server_get_channel_args(server), &tcp); + err = grpc_tcp_server_create(&exec_ctx, NULL, + grpc_server_get_channel_args(server), &tcp); if (err != GRPC_ERROR_NONE) { goto error; } @@ -148,9 +148,7 @@ int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) { const size_t naddrs = resolved->naddrs; errors = gpr_malloc(sizeof(*errors) * naddrs); for (i = 0; i < naddrs; i++) { - errors[i] = grpc_tcp_server_add_port( - tcp, (struct sockaddr *)&resolved->addrs[i].addr, - resolved->addrs[i].len, &port_temp); + errors[i] = grpc_tcp_server_add_port(tcp, &resolved->addrs[i], &port_temp); if (errors[i] == GRPC_ERROR_NONE) { if (port_num == -1) { port_num = port_temp; diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c index 9af17fb5ae..aa2ecf5743 100644 --- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c +++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c @@ -57,8 +57,12 @@ void grpc_server_add_insecure_channel_from_fd(grpc_server *server, char *name; gpr_asprintf(&name, "fd:%d", fd); - grpc_endpoint *server_endpoint = grpc_tcp_create( - grpc_fd_create(fd, name), GRPC_TCP_DEFAULT_READ_SLICE_SIZE, name); + grpc_resource_quota *resource_quota = grpc_resource_quota_from_channel_args( + grpc_server_get_channel_args(server)); + grpc_endpoint *server_endpoint = + grpc_tcp_create(grpc_fd_create(fd, name), resource_quota, + GRPC_TCP_DEFAULT_READ_SLICE_SIZE, name); + grpc_resource_quota_internal_unref(&exec_ctx, resource_quota); gpr_free(name); diff --git a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c index da3e284fcf..942638ad7f 100644 --- a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c +++ b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c @@ -61,13 +61,12 @@ typedef struct server_secure_state { grpc_server_credentials *creds; bool is_shutdown; gpr_mu mu; - gpr_refcount refcount; - grpc_closure destroy_closure; - grpc_closure *destroy_callback; + grpc_closure tcp_server_shutdown_complete; + grpc_closure *server_destroy_listener_done; } server_secure_state; typedef struct server_secure_connect { - server_secure_state *state; + server_secure_state *server_state; grpc_pollset *accepting_pollset; grpc_tcp_server_acceptor *acceptor; grpc_handshake_manager *handshake_mgr; @@ -77,39 +76,28 @@ typedef struct server_secure_connect { grpc_channel_args *args; } server_secure_connect; -static void state_ref(server_secure_state *state) { gpr_ref(&state->refcount); } - -static void state_unref(server_secure_state *state) { - if (gpr_unref(&state->refcount)) { - /* ensure all threads have unlocked */ - gpr_mu_lock(&state->mu); - gpr_mu_unlock(&state->mu); - /* clean up */ - GRPC_SECURITY_CONNECTOR_UNREF(&state->sc->base, "server"); - grpc_server_credentials_unref(state->creds); - gpr_free(state); - } -} - static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep, grpc_security_status status, grpc_endpoint *secure_endpoint, grpc_auth_context *auth_context) { - server_secure_connect *state = statep; + server_secure_connect *connection_state = statep; if (status == GRPC_SECURITY_OK) { if (secure_endpoint) { - gpr_mu_lock(&state->state->mu); - if (!state->state->is_shutdown) { + gpr_mu_lock(&connection_state->server_state->mu); + if (!connection_state->server_state->is_shutdown) { grpc_transport *transport = grpc_create_chttp2_transport( - exec_ctx, grpc_server_get_channel_args(state->state->server), + exec_ctx, grpc_server_get_channel_args( + connection_state->server_state->server), secure_endpoint, 0); grpc_arg args_to_add[2]; - args_to_add[0] = grpc_server_credentials_to_arg(state->state->creds); + args_to_add[0] = grpc_server_credentials_to_arg( + connection_state->server_state->creds); args_to_add[1] = grpc_auth_context_to_arg(auth_context); grpc_channel_args *args_copy = grpc_channel_args_copy_and_add( - state->args, args_to_add, GPR_ARRAY_SIZE(args_to_add)); - grpc_server_setup_transport(exec_ctx, state->state->server, transport, - state->accepting_pollset, args_copy); + connection_state->args, args_to_add, GPR_ARRAY_SIZE(args_to_add)); + grpc_server_setup_transport( + exec_ctx, connection_state->server_state->server, transport, + connection_state->accepting_pollset, args_copy); grpc_channel_args_destroy(args_copy); grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL); } else { @@ -117,21 +105,21 @@ static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep, * gone away. */ grpc_endpoint_destroy(exec_ctx, secure_endpoint); } - gpr_mu_unlock(&state->state->mu); + gpr_mu_unlock(&connection_state->server_state->mu); } } else { gpr_log(GPR_ERROR, "Secure transport failed with error %d", status); } - grpc_channel_args_destroy(state->args); - state_unref(state->state); - gpr_free(state); + grpc_channel_args_destroy(connection_state->args); + grpc_tcp_server_unref(exec_ctx, connection_state->server_state->tcp); + gpr_free(connection_state); } static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint, grpc_channel_args *args, - gpr_slice_buffer *read_buffer, void *user_data, + grpc_slice_buffer *read_buffer, void *user_data, grpc_error *error) { - server_secure_connect *state = user_data; + server_secure_connect *connection_state = user_data; if (error != GRPC_ERROR_NONE) { const char *error_str = grpc_error_string(error); gpr_log(GPR_ERROR, "Handshaking failed: %s", error_str); @@ -139,81 +127,107 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint, GRPC_ERROR_UNREF(error); grpc_channel_args_destroy(args); gpr_free(read_buffer); - grpc_handshake_manager_shutdown(exec_ctx, state->handshake_mgr); - grpc_handshake_manager_destroy(exec_ctx, state->handshake_mgr); - state_unref(state->state); - gpr_free(state); + grpc_handshake_manager_shutdown(exec_ctx, connection_state->handshake_mgr); + grpc_handshake_manager_destroy(exec_ctx, connection_state->handshake_mgr); + grpc_tcp_server_unref(exec_ctx, connection_state->server_state->tcp); + gpr_free(connection_state); return; } - grpc_handshake_manager_destroy(exec_ctx, state->handshake_mgr); - state->handshake_mgr = NULL; + grpc_handshake_manager_destroy(exec_ctx, connection_state->handshake_mgr); + connection_state->handshake_mgr = NULL; // TODO(roth, jboeuf): Convert security connector handshaking to use new // handshake API, and then move the code from on_secure_handshake_done() // into this function. - state->args = args; + connection_state->args = args; grpc_server_security_connector_do_handshake( - exec_ctx, state->state->sc, state->acceptor, endpoint, read_buffer, - state->deadline, on_secure_handshake_done, state); + exec_ctx, connection_state->server_state->sc, connection_state->acceptor, + endpoint, read_buffer, connection_state->deadline, + on_secure_handshake_done, connection_state); } static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp, grpc_pollset *accepting_pollset, grpc_tcp_server_acceptor *acceptor) { - server_secure_connect *state = gpr_malloc(sizeof(*state)); - state->state = statep; - state_ref(state->state); - state->accepting_pollset = accepting_pollset; - state->acceptor = acceptor; - state->handshake_mgr = grpc_handshake_manager_create(); + server_secure_state *server_state = statep; + server_secure_connect *connection_state = NULL; + gpr_mu_lock(&server_state->mu); + if (server_state->is_shutdown) { + gpr_mu_unlock(&server_state->mu); + grpc_endpoint_destroy(exec_ctx, tcp); + return; + } + gpr_mu_unlock(&server_state->mu); + grpc_tcp_server_ref(server_state->tcp); + connection_state = gpr_malloc(sizeof(*connection_state)); + connection_state->server_state = server_state; + connection_state->accepting_pollset = accepting_pollset; + connection_state->acceptor = acceptor; + connection_state->handshake_mgr = grpc_handshake_manager_create(); // TODO(roth): We should really get this timeout value from channel // args instead of hard-coding it. - state->deadline = gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), - gpr_time_from_seconds(120, GPR_TIMESPAN)); + connection_state->deadline = gpr_time_add( + gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(120, GPR_TIMESPAN)); grpc_handshake_manager_do_handshake( - exec_ctx, state->handshake_mgr, tcp, - grpc_server_get_channel_args(state->state->server), state->deadline, - acceptor, on_handshake_done, state); + exec_ctx, connection_state->handshake_mgr, tcp, + grpc_server_get_channel_args(connection_state->server_state->server), + connection_state->deadline, acceptor, on_handshake_done, + connection_state); } /* Server callback: start listening on our ports */ -static void start(grpc_exec_ctx *exec_ctx, grpc_server *server, void *statep, - grpc_pollset **pollsets, size_t pollset_count) { - server_secure_state *state = statep; - grpc_tcp_server_start(exec_ctx, state->tcp, pollsets, pollset_count, - on_accept, state); +static void server_start_listener(grpc_exec_ctx *exec_ctx, grpc_server *server, + void *statep, grpc_pollset **pollsets, + size_t pollset_count) { + server_secure_state *server_state = statep; + gpr_mu_lock(&server_state->mu); + server_state->is_shutdown = false; + gpr_mu_unlock(&server_state->mu); + grpc_tcp_server_start(exec_ctx, server_state->tcp, pollsets, pollset_count, + on_accept, server_state); } -static void destroy_done(grpc_exec_ctx *exec_ctx, void *statep, - grpc_error *error) { - server_secure_state *state = statep; - if (state->destroy_callback != NULL) { - state->destroy_callback->cb(exec_ctx, state->destroy_callback->cb_arg, - GRPC_ERROR_REF(error)); +static void tcp_server_shutdown_complete(grpc_exec_ctx *exec_ctx, void *statep, + grpc_error *error) { + server_secure_state *server_state = statep; + /* ensure all threads have unlocked */ + gpr_mu_lock(&server_state->mu); + grpc_closure *destroy_done = server_state->server_destroy_listener_done; + GPR_ASSERT(server_state->is_shutdown); + gpr_mu_unlock(&server_state->mu); + /* clean up */ + grpc_server_security_connector_shutdown(exec_ctx, server_state->sc); + + /* Flush queued work before a synchronous unref. */ + grpc_exec_ctx_flush(exec_ctx); + GRPC_SECURITY_CONNECTOR_UNREF(&server_state->sc->base, "server"); + grpc_server_credentials_unref(server_state->creds); + + if (destroy_done != NULL) { + destroy_done->cb(exec_ctx, destroy_done->cb_arg, GRPC_ERROR_REF(error)); + grpc_exec_ctx_flush(exec_ctx); } - grpc_server_security_connector_shutdown(exec_ctx, state->sc); - state_unref(state); + gpr_free(server_state); } -/* Server callback: destroy the tcp listener (so we don't generate further - callbacks) */ -static void destroy(grpc_exec_ctx *exec_ctx, grpc_server *server, void *statep, - grpc_closure *callback) { - server_secure_state *state = statep; +static void server_destroy_listener(grpc_exec_ctx *exec_ctx, + grpc_server *server, void *statep, + grpc_closure *callback) { + server_secure_state *server_state = statep; grpc_tcp_server *tcp; - gpr_mu_lock(&state->mu); - state->is_shutdown = true; - state->destroy_callback = callback; - tcp = state->tcp; - gpr_mu_unlock(&state->mu); + gpr_mu_lock(&server_state->mu); + server_state->is_shutdown = true; + server_state->server_destroy_listener_done = callback; + tcp = server_state->tcp; + gpr_mu_unlock(&server_state->mu); grpc_tcp_server_shutdown_listeners(exec_ctx, tcp); - grpc_tcp_server_unref(exec_ctx, tcp); + grpc_tcp_server_unref(exec_ctx, server_state->tcp); } int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr, grpc_server_credentials *creds) { grpc_resolved_addresses *resolved = NULL; grpc_tcp_server *tcp = NULL; - server_secure_state *state = NULL; + server_secure_state *server_state = NULL; size_t i; size_t count = 0; int port_num = -1; @@ -253,28 +267,27 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr, if (err != GRPC_ERROR_NONE) { goto error; } - state = gpr_malloc(sizeof(*state)); - memset(state, 0, sizeof(*state)); - grpc_closure_init(&state->destroy_closure, destroy_done, state); - err = grpc_tcp_server_create(&state->destroy_closure, + server_state = gpr_malloc(sizeof(*server_state)); + memset(server_state, 0, sizeof(*server_state)); + grpc_closure_init(&server_state->tcp_server_shutdown_complete, + tcp_server_shutdown_complete, server_state); + err = grpc_tcp_server_create(&exec_ctx, + &server_state->tcp_server_shutdown_complete, grpc_server_get_channel_args(server), &tcp); if (err != GRPC_ERROR_NONE) { goto error; } - state->server = server; - state->tcp = tcp; - state->sc = sc; - state->creds = grpc_server_credentials_ref(creds); - state->is_shutdown = false; - gpr_mu_init(&state->mu); - gpr_ref_init(&state->refcount, 1); + server_state->server = server; + server_state->tcp = tcp; + server_state->sc = sc; + server_state->creds = grpc_server_credentials_ref(creds); + server_state->is_shutdown = true; + gpr_mu_init(&server_state->mu); errors = gpr_malloc(sizeof(*errors) * resolved->naddrs); for (i = 0; i < resolved->naddrs; i++) { - errors[i] = grpc_tcp_server_add_port( - tcp, (struct sockaddr *)&resolved->addrs[i].addr, - resolved->addrs[i].len, &port_temp); + errors[i] = grpc_tcp_server_add_port(tcp, &resolved->addrs[i], &port_temp); if (errors[i] == GRPC_ERROR_NONE) { if (port_num == -1) { port_num = port_temp; @@ -313,7 +326,8 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr, grpc_resolved_addresses_destroy(resolved); /* Register with the server only upon success */ - grpc_server_add_listener(&exec_ctx, server, state, start, destroy); + grpc_server_add_listener(&exec_ctx, server, server_state, + server_start_listener, server_destroy_listener); grpc_exec_ctx_finish(&exec_ctx); return port_num; @@ -334,10 +348,11 @@ error: grpc_tcp_server_unref(&exec_ctx, tcp); } else { if (sc) { + grpc_exec_ctx_flush(&exec_ctx); GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "server"); } - if (state) { - gpr_free(state); + if (server_state) { + gpr_free(server_state); } } grpc_exec_ctx_finish(&exec_ctx); diff --git a/src/core/ext/transport/chttp2/transport/bin_decoder.c b/src/core/ext/transport/chttp2/transport/bin_decoder.c index 2d90b01cd8..3eef80b557 100644 --- a/src/core/ext/transport/chttp2/transport/bin_decoder.c +++ b/src/core/ext/transport/chttp2/transport/bin_decoder.c @@ -34,6 +34,7 @@ #include "src/core/ext/transport/chttp2/transport/bin_decoder.h" #include <grpc/support/alloc.h> #include <grpc/support/log.h> +#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" static uint8_t decode_table[] = { @@ -142,11 +143,11 @@ bool grpc_base64_decode_partial(struct grpc_base64_decode_context *ctx) { return true; } -gpr_slice grpc_chttp2_base64_decode(gpr_slice input) { - size_t input_length = GPR_SLICE_LENGTH(input); +grpc_slice grpc_chttp2_base64_decode(grpc_slice input) { + size_t input_length = GRPC_SLICE_LENGTH(input); size_t output_length = input_length / 4 * 3; struct grpc_base64_decode_context ctx; - gpr_slice output; + grpc_slice output; if (input_length % 4 != 0) { gpr_log(GPR_ERROR, @@ -158,7 +159,7 @@ gpr_slice grpc_chttp2_base64_decode(gpr_slice input) { } if (input_length > 0) { - uint8_t *input_end = GPR_SLICE_END_PTR(input); + uint8_t *input_end = GRPC_SLICE_END_PTR(input); if (*(--input_end) == '=') { output_length--; if (*(--input_end) == '=') { @@ -166,30 +167,30 @@ gpr_slice grpc_chttp2_base64_decode(gpr_slice input) { } } } - output = gpr_slice_malloc(output_length); + output = grpc_slice_malloc(output_length); - ctx.input_cur = GPR_SLICE_START_PTR(input); - ctx.input_end = GPR_SLICE_END_PTR(input); - ctx.output_cur = GPR_SLICE_START_PTR(output); - ctx.output_end = GPR_SLICE_END_PTR(output); + ctx.input_cur = GRPC_SLICE_START_PTR(input); + ctx.input_end = GRPC_SLICE_END_PTR(input); + ctx.output_cur = GRPC_SLICE_START_PTR(output); + ctx.output_end = GRPC_SLICE_END_PTR(output); ctx.contains_tail = false; if (!grpc_base64_decode_partial(&ctx)) { - char *s = gpr_dump_slice(input, GPR_DUMP_ASCII); + char *s = grpc_dump_slice(input, GPR_DUMP_ASCII); gpr_log(GPR_ERROR, "Base64 decoding failed, input string:\n%s\n", s); gpr_free(s); - gpr_slice_unref(output); + grpc_slice_unref(output); return gpr_empty_slice(); } - GPR_ASSERT(ctx.output_cur == GPR_SLICE_END_PTR(output)); - GPR_ASSERT(ctx.input_cur == GPR_SLICE_END_PTR(input)); + GPR_ASSERT(ctx.output_cur == GRPC_SLICE_END_PTR(output)); + GPR_ASSERT(ctx.input_cur == GRPC_SLICE_END_PTR(input)); return output; } -gpr_slice grpc_chttp2_base64_decode_with_length(gpr_slice input, - size_t output_length) { - size_t input_length = GPR_SLICE_LENGTH(input); - gpr_slice output = gpr_slice_malloc(output_length); +grpc_slice grpc_chttp2_base64_decode_with_length(grpc_slice input, + size_t output_length) { + size_t input_length = GRPC_SLICE_LENGTH(input); + grpc_slice output = grpc_slice_malloc(output_length); struct grpc_base64_decode_context ctx; // The length of a base64 string cannot be 4 * n + 1 @@ -199,7 +200,7 @@ gpr_slice grpc_chttp2_base64_decode_with_length(gpr_slice input, "grpc_chttp2_base64_decode_with_length has a length of %d, which " "has a tail of 1 byte.\n", (int)input_length); - gpr_slice_unref(output); + grpc_slice_unref(output); return gpr_empty_slice(); } @@ -209,24 +210,24 @@ gpr_slice grpc_chttp2_base64_decode_with_length(gpr_slice input, "than the max possible output length %d.\n", (int)output_length, (int)(input_length / 4 * 3 + tail_xtra[input_length % 4])); - gpr_slice_unref(output); + grpc_slice_unref(output); return gpr_empty_slice(); } - ctx.input_cur = GPR_SLICE_START_PTR(input); - ctx.input_end = GPR_SLICE_END_PTR(input); - ctx.output_cur = GPR_SLICE_START_PTR(output); - ctx.output_end = GPR_SLICE_END_PTR(output); + ctx.input_cur = GRPC_SLICE_START_PTR(input); + ctx.input_end = GRPC_SLICE_END_PTR(input); + ctx.output_cur = GRPC_SLICE_START_PTR(output); + ctx.output_end = GRPC_SLICE_END_PTR(output); ctx.contains_tail = true; if (!grpc_base64_decode_partial(&ctx)) { - char *s = gpr_dump_slice(input, GPR_DUMP_ASCII); + char *s = grpc_dump_slice(input, GPR_DUMP_ASCII); gpr_log(GPR_ERROR, "Base64 decoding failed, input string:\n%s\n", s); gpr_free(s); - gpr_slice_unref(output); + grpc_slice_unref(output); return gpr_empty_slice(); } - GPR_ASSERT(ctx.output_cur == GPR_SLICE_END_PTR(output)); - GPR_ASSERT(ctx.input_cur <= GPR_SLICE_END_PTR(input)); + GPR_ASSERT(ctx.output_cur == GRPC_SLICE_END_PTR(output)); + GPR_ASSERT(ctx.input_cur <= GRPC_SLICE_END_PTR(input)); return output; } diff --git a/src/core/ext/transport/chttp2/transport/bin_decoder.h b/src/core/ext/transport/chttp2/transport/bin_decoder.h index b9d40c9b74..83a90be519 100644 --- a/src/core/ext/transport/chttp2/transport/bin_decoder.h +++ b/src/core/ext/transport/chttp2/transport/bin_decoder.h @@ -34,7 +34,7 @@ #ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_DECODER_H #define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_DECODER_H -#include <grpc/support/slice.h> +#include <grpc/slice.h> #include <stdbool.h> struct grpc_base64_decode_context { @@ -55,12 +55,12 @@ bool grpc_base64_decode_partial(struct grpc_base64_decode_context *ctx); /* base64 decode a slice with pad chars. Returns a new slice, does not take ownership of the input. Returns an empty slice if decoding is failed. */ -gpr_slice grpc_chttp2_base64_decode(gpr_slice input); +grpc_slice grpc_chttp2_base64_decode(grpc_slice input); /* base64 decode a slice without pad chars, data length is needed. Returns a new slice, does not take ownership of the input. Returns an empty slice if decoding is failed. */ -gpr_slice grpc_chttp2_base64_decode_with_length(gpr_slice input, - size_t output_length); +grpc_slice grpc_chttp2_base64_decode_with_length(grpc_slice input, + size_t output_length); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_DECODER_H */ diff --git a/src/core/ext/transport/chttp2/transport/bin_encoder.c b/src/core/ext/transport/chttp2/transport/bin_encoder.c index 1b43c28be1..af25a4352a 100644 --- a/src/core/ext/transport/chttp2/transport/bin_encoder.c +++ b/src/core/ext/transport/chttp2/transport/bin_encoder.c @@ -61,14 +61,14 @@ static const b64_huff_sym huff_alphabet[64] = { static const uint8_t tail_xtra[3] = {0, 2, 3}; -gpr_slice grpc_chttp2_base64_encode(gpr_slice input) { - size_t input_length = GPR_SLICE_LENGTH(input); +grpc_slice grpc_chttp2_base64_encode(grpc_slice input) { + size_t input_length = GRPC_SLICE_LENGTH(input); size_t input_triplets = input_length / 3; size_t tail_case = input_length % 3; size_t output_length = input_triplets * 4 + tail_xtra[tail_case]; - gpr_slice output = gpr_slice_malloc(output_length); - uint8_t *in = GPR_SLICE_START_PTR(input); - char *out = (char *)GPR_SLICE_START_PTR(output); + grpc_slice output = grpc_slice_malloc(output_length); + uint8_t *in = GRPC_SLICE_START_PTR(input); + char *out = (char *)GRPC_SLICE_START_PTR(output); size_t i; /* encode full triplets */ @@ -100,27 +100,29 @@ gpr_slice grpc_chttp2_base64_encode(gpr_slice input) { break; } - GPR_ASSERT(out == (char *)GPR_SLICE_END_PTR(output)); - GPR_ASSERT(in == GPR_SLICE_END_PTR(input)); + GPR_ASSERT(out == (char *)GRPC_SLICE_END_PTR(output)); + GPR_ASSERT(in == GRPC_SLICE_END_PTR(input)); return output; } -gpr_slice grpc_chttp2_huffman_compress(gpr_slice input) { +grpc_slice grpc_chttp2_huffman_compress(grpc_slice input) { size_t nbits; uint8_t *in; uint8_t *out; - gpr_slice output; + grpc_slice output; uint32_t temp = 0; uint32_t temp_length = 0; nbits = 0; - for (in = GPR_SLICE_START_PTR(input); in != GPR_SLICE_END_PTR(input); ++in) { + for (in = GRPC_SLICE_START_PTR(input); in != GRPC_SLICE_END_PTR(input); + ++in) { nbits += grpc_chttp2_huffsyms[*in].length; } - output = gpr_slice_malloc(nbits / 8 + (nbits % 8 != 0)); - out = GPR_SLICE_START_PTR(output); - for (in = GPR_SLICE_START_PTR(input); in != GPR_SLICE_END_PTR(input); ++in) { + output = grpc_slice_malloc(nbits / 8 + (nbits % 8 != 0)); + out = GRPC_SLICE_START_PTR(output); + for (in = GRPC_SLICE_START_PTR(input); in != GRPC_SLICE_END_PTR(input); + ++in) { int sym = *in; temp <<= grpc_chttp2_huffsyms[sym].length; temp |= grpc_chttp2_huffsyms[sym].bits; @@ -141,7 +143,7 @@ gpr_slice grpc_chttp2_huffman_compress(gpr_slice input) { (uint8_t)(0xffu >> temp_length)); } - GPR_ASSERT(out == GPR_SLICE_END_PTR(output)); + GPR_ASSERT(out == GRPC_SLICE_END_PTR(output)); return output; } @@ -175,16 +177,17 @@ static void enc_add1(huff_out *out, uint8_t a) { enc_flush_some(out); } -gpr_slice grpc_chttp2_base64_encode_and_huffman_compress_impl(gpr_slice input) { - size_t input_length = GPR_SLICE_LENGTH(input); +grpc_slice grpc_chttp2_base64_encode_and_huffman_compress_impl( + grpc_slice input) { + size_t input_length = GRPC_SLICE_LENGTH(input); size_t input_triplets = input_length / 3; size_t tail_case = input_length % 3; size_t output_syms = input_triplets * 4 + tail_xtra[tail_case]; size_t max_output_bits = 11 * output_syms; size_t max_output_length = max_output_bits / 8 + (max_output_bits % 8 != 0); - gpr_slice output = gpr_slice_malloc(max_output_length); - uint8_t *in = GPR_SLICE_START_PTR(input); - uint8_t *start_out = GPR_SLICE_START_PTR(output); + grpc_slice output = grpc_slice_malloc(max_output_length); + uint8_t *in = GRPC_SLICE_START_PTR(input); + uint8_t *start_out = GRPC_SLICE_START_PTR(output); huff_out out; size_t i; @@ -231,9 +234,9 @@ gpr_slice grpc_chttp2_base64_encode_and_huffman_compress_impl(gpr_slice input) { (uint8_t)(0xffu >> out.temp_length)); } - GPR_ASSERT(out.out <= GPR_SLICE_END_PTR(output)); - GPR_SLICE_SET_LENGTH(output, out.out - start_out); + GPR_ASSERT(out.out <= GRPC_SLICE_END_PTR(output)); + GRPC_SLICE_SET_LENGTH(output, out.out - start_out); - GPR_ASSERT(in == GPR_SLICE_END_PTR(input)); + GPR_ASSERT(in == GRPC_SLICE_END_PTR(input)); return output; } diff --git a/src/core/ext/transport/chttp2/transport/bin_encoder.h b/src/core/ext/transport/chttp2/transport/bin_encoder.h index 61ebbafa9a..9e143b46e2 100644 --- a/src/core/ext/transport/chttp2/transport/bin_encoder.h +++ b/src/core/ext/transport/chttp2/transport/bin_encoder.h @@ -34,21 +34,22 @@ #ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_ENCODER_H #define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_ENCODER_H -#include <grpc/support/slice.h> +#include <grpc/slice.h> /* base64 encode a slice. Returns a new slice, does not take ownership of the input */ -gpr_slice grpc_chttp2_base64_encode(gpr_slice input); +grpc_slice grpc_chttp2_base64_encode(grpc_slice input); /* Compress a slice with the static huffman encoder detailed in the hpack standard. Returns a new slice, does not take ownership of the input */ -gpr_slice grpc_chttp2_huffman_compress(gpr_slice input); +grpc_slice grpc_chttp2_huffman_compress(grpc_slice input); /* equivalent to: - gpr_slice x = grpc_chttp2_base64_encode(input); - gpr_slice y = grpc_chttp2_huffman_compress(x); - gpr_slice_unref(x); + grpc_slice x = grpc_chttp2_base64_encode(input); + grpc_slice y = grpc_chttp2_huffman_compress(x); + grpc_slice_unref(x); return y; */ -gpr_slice grpc_chttp2_base64_encode_and_huffman_compress_impl(gpr_slice input); +grpc_slice grpc_chttp2_base64_encode_and_huffman_compress_impl( + grpc_slice input); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_BIN_ENCODER_H */ diff --git a/src/core/ext/transport/chttp2/transport/chttp2_plugin.c b/src/core/ext/transport/chttp2/transport/chttp2_plugin.c index 7d5279b9da..bd87253ed3 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_plugin.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_plugin.c @@ -36,14 +36,11 @@ #include "src/core/lib/debug/trace.h" #include "src/core/lib/transport/metadata.h" -extern int grpc_http_write_state_trace; - void grpc_chttp2_plugin_init(void) { grpc_chttp2_base64_encode_and_huffman_compress = grpc_chttp2_base64_encode_and_huffman_compress_impl; grpc_register_tracer("http", &grpc_http_trace); grpc_register_tracer("flowctl", &grpc_flowctl_trace); - grpc_register_tracer("http_write_state", &grpc_http_write_state_trace); } void grpc_chttp2_plugin_shutdown(void) {} diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 1dd7fef76f..1ffa9165b2 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -38,9 +38,9 @@ #include <stdio.h> #include <string.h> +#include <grpc/slice_buffer.h> #include <grpc/support/alloc.h> #include <grpc/support/log.h> -#include <grpc/support/slice_buffer.h> #include <grpc/support/string_util.h> #include <grpc/support/useful.h> @@ -51,6 +51,7 @@ #include "src/core/lib/http/parser.h" #include "src/core/lib/iomgr/workqueue.h" #include "src/core/lib/profiling/timers.h" +#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" #include "src/core/lib/transport/static_metadata.h" #include "src/core/lib/transport/timeout_encoding.h" @@ -65,91 +66,73 @@ #define MAX_CLIENT_STREAM_ID 0x7fffffffu int grpc_http_trace = 0; int grpc_flowctl_trace = 0; -int grpc_http_write_state_trace = 0; - -#define TRANSPORT_FROM_WRITING(tw) \ - ((grpc_chttp2_transport *)((char *)(tw)-offsetof(grpc_chttp2_transport, \ - writing))) - -#define TRANSPORT_FROM_PARSING(tp) \ - ((grpc_chttp2_transport *)((char *)(tp)-offsetof(grpc_chttp2_transport, \ - parsing))) - -#define TRANSPORT_FROM_GLOBAL(tg) \ - ((grpc_chttp2_transport *)((char *)(tg)-offsetof(grpc_chttp2_transport, \ - global))) - -#define STREAM_FROM_GLOBAL(sg) \ - ((grpc_chttp2_stream *)((char *)(sg)-offsetof(grpc_chttp2_stream, global))) - -#define STREAM_FROM_PARSING(sg) \ - ((grpc_chttp2_stream *)((char *)(sg)-offsetof(grpc_chttp2_stream, parsing))) static const grpc_transport_vtable vtable; /* forward declarations of various callbacks that we'll build closures around */ -static void writing_action(grpc_exec_ctx *exec_ctx, void *t, grpc_error *error); -static void reading_action(grpc_exec_ctx *exec_ctx, void *t, grpc_error *error); -static void parsing_action(grpc_exec_ctx *exec_ctx, void *t, grpc_error *error); -static void reading_action_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error); -static void post_parse_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error); -static void initiate_writing_locked(grpc_exec_ctx *exec_ctx, void *t, +static void write_action_begin_locked(grpc_exec_ctx *exec_ctx, void *t, + grpc_error *error); +static void write_action(grpc_exec_ctx *exec_ctx, void *t, grpc_error *error); +static void write_action_end(grpc_exec_ctx *exec_ctx, void *t, + grpc_error *error); +static void write_action_end_locked(grpc_exec_ctx *exec_ctx, void *t, grpc_error *error); -static void initiate_read_flush_locked(grpc_exec_ctx *exec_ctx, void *t, - grpc_error *error); -static void terminate_writing_with_lock(grpc_exec_ctx *exec_ctx, void *t, - grpc_error *error); -static void start_writing(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t); -static void end_waiting_for_write(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, grpc_error *error); +static void read_action_begin(grpc_exec_ctx *exec_ctx, void *t, + grpc_error *error); +static void read_action_locked(grpc_exec_ctx *exec_ctx, void *t, + grpc_error *error); +static void complete_fetch_locked(grpc_exec_ctx *exec_ctx, void *gs, + grpc_error *error); +static void complete_fetch(grpc_exec_ctx *exec_ctx, void *gs, + grpc_error *error); /** Set a transport level setting, and push it to our peer */ static void push_setting(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_setting_id id, uint32_t value); -/** Start disconnection chain */ -static void drop_connection(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, - grpc_error *error); - -/** Cancel a stream: coming from the transport API */ -static void cancel_from_api(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, - grpc_error *error); - -static void close_from_api(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, - grpc_error *error); +static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_stream *s, grpc_error *error); /** Start new streams that have been created if we can */ -static void maybe_start_some_streams( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global); - -static void connectivity_state_set( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_connectivity_state state, grpc_error *error, const char *reason); +static void maybe_start_some_streams(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); -static void check_read_ops(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global); - -static void incoming_byte_stream_update_flow_control( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, size_t max_size_hint, - size_t have_already); +static void connectivity_state_set(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_connectivity_state state, + grpc_error *error, const char *reason); + +static void incoming_byte_stream_update_flow_control(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + size_t max_size_hint, + size_t have_already); static void incoming_byte_stream_destroy_locked(grpc_exec_ctx *exec_ctx, void *byte_stream, grpc_error *error_ignored); static void fail_pending_writes(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, + grpc_chttp2_transport *t, grpc_chttp2_stream *s, grpc_error *error); -static void set_write_state(grpc_chttp2_transport *t, - grpc_chttp2_write_state state, const char *reason); +static void benign_reclaimer(grpc_exec_ctx *exec_ctx, void *t, + grpc_error *error); +static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *t, + grpc_error *error); +static void destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *t, + grpc_error *error); +static void destructive_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *t, + grpc_error *error); + +static void post_benign_reclaimer(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); +static void post_destructive_reclaimer(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); + +static void close_transport_locked(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, grpc_error *error); +static void end_all_the_calls(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_error *error); /******************************************************************************* * CONSTRUCTION/DESTRUCTION/REFCOUNTING @@ -161,34 +144,31 @@ static void destruct_transport(grpc_exec_ctx *exec_ctx, grpc_endpoint_destroy(exec_ctx, t->ep); - gpr_slice_buffer_destroy(&t->global.qbuf); + grpc_slice_buffer_destroy(&t->qbuf); - gpr_slice_buffer_destroy(&t->writing.outbuf); - grpc_chttp2_hpack_compressor_destroy(&t->writing.hpack_compressor); + grpc_slice_buffer_destroy(&t->outbuf); + grpc_chttp2_hpack_compressor_destroy(&t->hpack_compressor); - gpr_slice_buffer_destroy(&t->parsing.qbuf); - gpr_slice_buffer_destroy(&t->read_buffer); - grpc_chttp2_hpack_parser_destroy(&t->parsing.hpack_parser); - grpc_chttp2_goaway_parser_destroy(&t->parsing.goaway_parser); + grpc_slice_buffer_destroy(&t->read_buffer); + grpc_chttp2_hpack_parser_destroy(&t->hpack_parser); + grpc_chttp2_goaway_parser_destroy(&t->goaway_parser); for (i = 0; i < STREAM_LIST_COUNT; i++) { GPR_ASSERT(t->lists[i].head == NULL); GPR_ASSERT(t->lists[i].tail == NULL); } - GPR_ASSERT(grpc_chttp2_stream_map_size(&t->parsing_stream_map) == 0); - GPR_ASSERT(grpc_chttp2_stream_map_size(&t->new_stream_map) == 0); + GPR_ASSERT(grpc_chttp2_stream_map_size(&t->stream_map) == 0); - grpc_chttp2_stream_map_destroy(&t->parsing_stream_map); - grpc_chttp2_stream_map_destroy(&t->new_stream_map); + grpc_chttp2_stream_map_destroy(&t->stream_map); grpc_connectivity_state_destroy(exec_ctx, &t->channel_callback.state_tracker); - grpc_combiner_destroy(exec_ctx, t->executor.combiner); + grpc_combiner_destroy(exec_ctx, t->combiner); /* callback remaining pings: they're not allowed to call into the transpot, and maybe they hold resources that need to be freed */ - while (t->global.pings.next != &t->global.pings) { - grpc_chttp2_outstanding_ping *ping = t->global.pings.next; + while (t->pings.next != &t->pings) { + grpc_chttp2_outstanding_ping *ping = t->pings.next; grpc_exec_ctx_sched(exec_ctx, ping->on_recv, GRPC_ERROR_CREATE("Transport closed"), NULL); ping->next->prev = ping->prev; @@ -196,37 +176,40 @@ static void destruct_transport(grpc_exec_ctx *exec_ctx, gpr_free(ping); } + while (t->write_cb_pool) { + grpc_chttp2_write_cb *next = t->write_cb_pool->next; + gpr_free(t->write_cb_pool); + t->write_cb_pool = next; + } + gpr_free(t->peer_string); gpr_free(t); } -/*#define REFCOUNTING_DEBUG 1*/ -#ifdef REFCOUNTING_DEBUG -#define REF_TRANSPORT(t, r) ref_transport(t, r, __FILE__, __LINE__) -#define UNREF_TRANSPORT(cl, t, r) unref_transport(cl, t, r, __FILE__, __LINE__) -static void unref_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, - const char *reason, const char *file, int line) { - gpr_log(GPR_DEBUG, "chttp2:unref:%p %d->%d %s [%s:%d]", t, t->refs.count, - t->refs.count - 1, reason, file, line); +#ifdef GRPC_CHTTP2_REFCOUNTING_DEBUG +void grpc_chttp2_unref_transport(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, const char *reason, + const char *file, int line) { + gpr_log(GPR_DEBUG, "chttp2:unref:%p %" PRIdPTR "->%" PRIdPTR " %s [%s:%d]", t, + t->refs.count, t->refs.count - 1, reason, file, line); if (!gpr_unref(&t->refs)) return; destruct_transport(exec_ctx, t); } -static void ref_transport(grpc_chttp2_transport *t, const char *reason, - const char *file, int line) { - gpr_log(GPR_DEBUG, "chttp2: ref:%p %d->%d %s [%s:%d]", t, t->refs.count, - t->refs.count + 1, reason, file, line); +void grpc_chttp2_ref_transport(grpc_chttp2_transport *t, const char *reason, + const char *file, int line) { + gpr_log(GPR_DEBUG, "chttp2: ref:%p %" PRIdPTR "->%" PRIdPTR " %s [%s:%d]", t, + t->refs.count, t->refs.count + 1, reason, file, line); gpr_ref(&t->refs); } #else -#define REF_TRANSPORT(t, r) ref_transport(t) -#define UNREF_TRANSPORT(cl, t, r) unref_transport(cl, t) -static void unref_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { +void grpc_chttp2_unref_transport(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { if (!gpr_unref(&t->refs)) return; destruct_transport(exec_ctx, t); } -static void ref_transport(grpc_chttp2_transport *t) { gpr_ref(&t->refs); } +void grpc_chttp2_ref_transport(grpc_chttp2_transport *t) { gpr_ref(&t->refs); } #endif static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, @@ -241,81 +224,72 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, memset(t, 0, sizeof(*t)); t->base.vtable = &vtable; - t->executor.write_state = GRPC_CHTTP2_WRITES_CORKED; t->ep = ep; /* one ref is for destroy */ gpr_ref_init(&t->refs, 1); - /* ref is dropped at transport close() */ - gpr_ref_init(&t->shutdown_ep_refs, 1); - t->executor.combiner = grpc_combiner_create(grpc_endpoint_get_workqueue(ep)); + t->combiner = grpc_combiner_create(grpc_endpoint_get_workqueue(ep)); t->peer_string = grpc_endpoint_get_peer(ep); t->endpoint_reading = 1; - t->global.next_stream_id = is_client ? 1 : 2; - t->global.is_client = is_client; - t->writing.outgoing_window = DEFAULT_WINDOW; - t->parsing.incoming_window = DEFAULT_WINDOW; - t->global.stream_lookahead = DEFAULT_WINDOW; - t->global.connection_window_target = DEFAULT_CONNECTION_WINDOW_TARGET; - t->global.ping_counter = 1; - t->global.pings.next = t->global.pings.prev = &t->global.pings; - t->parsing.is_client = is_client; - t->parsing.deframe_state = - is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0; - t->parsing.is_first_frame = true; - t->writing.is_client = is_client; + t->next_stream_id = is_client ? 1 : 2; + t->is_client = is_client; + t->outgoing_window = DEFAULT_WINDOW; + t->incoming_window = DEFAULT_WINDOW; + t->stream_lookahead = DEFAULT_WINDOW; + t->connection_window_target = DEFAULT_CONNECTION_WINDOW_TARGET; + t->ping_counter = 1; + t->pings.next = t->pings.prev = &t->pings; + t->deframe_state = is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0; + t->is_first_frame = true; grpc_connectivity_state_init( &t->channel_callback.state_tracker, GRPC_CHANNEL_READY, is_client ? "client_transport" : "server_transport"); - gpr_slice_buffer_init(&t->global.qbuf); - - gpr_slice_buffer_init(&t->writing.outbuf); - grpc_chttp2_hpack_compressor_init(&t->writing.hpack_compressor); - grpc_closure_init(&t->writing_action, writing_action, t); - grpc_closure_init(&t->reading_action, reading_action, t); - grpc_closure_init(&t->reading_action_locked, reading_action_locked, t); - grpc_closure_init(&t->parsing_action, parsing_action, t); - grpc_closure_init(&t->post_parse_locked, post_parse_locked, t); - grpc_closure_init(&t->initiate_writing, initiate_writing_locked, t); - grpc_closure_init(&t->terminate_writing, terminate_writing_with_lock, t); - grpc_closure_init(&t->initiate_read_flush_locked, initiate_read_flush_locked, - t); - grpc_closure_init(&t->writing.done_cb, grpc_chttp2_terminate_writing, - &t->writing); + grpc_slice_buffer_init(&t->qbuf); - gpr_slice_buffer_init(&t->parsing.qbuf); - grpc_chttp2_goaway_parser_init(&t->parsing.goaway_parser); - grpc_chttp2_hpack_parser_init(&t->parsing.hpack_parser); + grpc_slice_buffer_init(&t->outbuf); + grpc_chttp2_hpack_compressor_init(&t->hpack_compressor); - gpr_slice_buffer_init(&t->read_buffer); + grpc_closure_init(&t->write_action_begin_locked, write_action_begin_locked, + t); + grpc_closure_init(&t->write_action, write_action, t); + grpc_closure_init(&t->write_action_end, write_action_end, t); + grpc_closure_init(&t->write_action_end_locked, write_action_end_locked, t); + grpc_closure_init(&t->read_action_begin, read_action_begin, t); + grpc_closure_init(&t->read_action_locked, read_action_locked, t); + grpc_closure_init(&t->benign_reclaimer, benign_reclaimer, t); + grpc_closure_init(&t->destructive_reclaimer, destructive_reclaimer, t); + grpc_closure_init(&t->benign_reclaimer_locked, benign_reclaimer_locked, t); + grpc_closure_init(&t->destructive_reclaimer_locked, + destructive_reclaimer_locked, t); + + grpc_chttp2_goaway_parser_init(&t->goaway_parser); + grpc_chttp2_hpack_parser_init(&t->hpack_parser); + + grpc_slice_buffer_init(&t->read_buffer); /* 8 is a random stab in the dark as to a good initial size: it's small enough that it shouldn't waste memory for infrequently used connections, yet large enough that the exponential growth should happen nicely when it's needed. TODO(ctiller): tune this */ - grpc_chttp2_stream_map_init(&t->parsing_stream_map, 8); - grpc_chttp2_stream_map_init(&t->new_stream_map, 8); + grpc_chttp2_stream_map_init(&t->stream_map, 8); /* copy in initial settings to all setting sets */ for (i = 0; i < GRPC_CHTTP2_NUM_SETTINGS; i++) { - t->parsing.settings[i] = grpc_chttp2_settings_parameters[i].default_value; for (j = 0; j < GRPC_NUM_SETTING_SETS; j++) { - t->global.settings[j][i] = - grpc_chttp2_settings_parameters[i].default_value; + t->settings[j][i] = grpc_chttp2_settings_parameters[i].default_value; } } - t->global.dirtied_local_settings = 1; + t->dirtied_local_settings = 1; /* Hack: it's common for implementations to assume 65536 bytes initial send window -- this should by rights be 0 */ - t->global.force_send_settings = 1 << GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; - t->global.sent_local_settings = 0; + t->force_send_settings = 1 << GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; + t->sent_local_settings = 0; if (is_client) { - gpr_slice_buffer_add( - &t->writing.outbuf, - gpr_slice_from_copied_string(GRPC_CHTTP2_CLIENT_CONNECT_STRING)); - grpc_chttp2_initiate_write(exec_ctx, &t->global, false, "initial_write"); + grpc_slice_buffer_add(&t->outbuf, grpc_slice_from_copied_string( + GRPC_CHTTP2_CLIENT_CONNECT_STRING)); + grpc_chttp2_initiate_write(exec_ctx, t, false, "initial_write"); } /* configure http2 the way we like it */ @@ -330,34 +304,18 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, if (channel_args) { for (i = 0; i < channel_args->num_args; i++) { - if (0 == - strcmp(channel_args->args[i].key, GRPC_ARG_MAX_CONCURRENT_STREAMS)) { - if (is_client) { - gpr_log(GPR_ERROR, "%s: is ignored on the client", - GRPC_ARG_MAX_CONCURRENT_STREAMS); - } else { - const grpc_integer_options options = {-1, 0, INT_MAX}; - const int value = - grpc_channel_arg_get_integer(&channel_args->args[i], options); - if (value >= 0) { - push_setting(exec_ctx, t, - GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, - (uint32_t)value); - } - } - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_HTTP2_INITIAL_SEQUENCE_NUMBER)) { + if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_INITIAL_SEQUENCE_NUMBER)) { const grpc_integer_options options = {-1, 0, INT_MAX}; const int value = grpc_channel_arg_get_integer(&channel_args->args[i], options); if (value >= 0) { - if ((t->global.next_stream_id & 1) != (value & 1)) { + if ((t->next_stream_id & 1) != (value & 1)) { gpr_log(GPR_ERROR, "%s: low bit must be %d on %s", GRPC_ARG_HTTP2_INITIAL_SEQUENCE_NUMBER, - t->global.next_stream_id & 1, - is_client ? "client" : "server"); + t->next_stream_id & 1, is_client ? "client" : "server"); } else { - t->global.next_stream_id = (uint32_t)value; + t->next_stream_id = (uint32_t)value; } } } else if (0 == strcmp(channel_args->args[i].key, @@ -366,16 +324,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, const int value = grpc_channel_arg_get_integer(&channel_args->args[i], options); if (value >= 0) { - t->global.stream_lookahead = (uint32_t)value; - } - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_DECODER)) { - const grpc_integer_options options = {-1, 0, INT_MAX}; - const int value = - grpc_channel_arg_get_integer(&channel_args->args[i], options); - if (value >= 0) { - push_setting(exec_ctx, t, GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE, - (uint32_t)value); + t->stream_lookahead = (uint32_t)value; } } else if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_ENCODER)) { @@ -383,118 +332,126 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, const int value = grpc_channel_arg_get_integer(&channel_args->args[i], options); if (value >= 0) { - grpc_chttp2_hpack_compressor_set_max_usable_size( - &t->writing.hpack_compressor, (uint32_t)value); + grpc_chttp2_hpack_compressor_set_max_usable_size(&t->hpack_compressor, + (uint32_t)value); } - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_MAX_METADATA_SIZE)) { - const grpc_integer_options options = {-1, 0, INT_MAX}; - const int value = - grpc_channel_arg_get_integer(&channel_args->args[i], options); - if (value >= 0) { - push_setting(exec_ctx, t, GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE, - (uint32_t)value); - } - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_HTTP2_MAX_FRAME_SIZE)) { - if (channel_args->args[i].type != GRPC_ARG_INTEGER) { - gpr_log(GPR_ERROR, "%s: must be an integer", - GRPC_ARG_HTTP2_MAX_FRAME_SIZE); - } else if (channel_args->args[i].value.integer < 16384 || - channel_args->args[i].value.integer > 16777215) { - gpr_log(GPR_ERROR, "%s: must be between 16384 and 16777215", - GRPC_ARG_HTTP2_MAX_FRAME_SIZE); - } else { - push_setting(exec_ctx, t, GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE, - (uint32_t)channel_args->args[i].value.integer); + } else { + static const struct { + const char *channel_arg_name; + grpc_chttp2_setting_id setting_id; + grpc_integer_options integer_options; + bool availability[2] /* server, client */; + } settings_map[] = { + {GRPC_ARG_MAX_CONCURRENT_STREAMS, + GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, + {-1, 0, INT_MAX}, + {true, false}}, + {GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_DECODER, + GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE, + {-1, 0, INT_MAX}, + {true, true}}, + {GRPC_ARG_MAX_METADATA_SIZE, + GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE, + {-1, 0, INT_MAX}, + {true, true}}, + {GRPC_ARG_HTTP2_MAX_FRAME_SIZE, + GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE, + {-1, 16384, 16777215}, + {true, true}}, + }; + for (j = 0; j < (int)GPR_ARRAY_SIZE(settings_map); j++) { + if (0 == strcmp(channel_args->args[i].key, + settings_map[j].channel_arg_name)) { + if (!settings_map[j].availability[is_client]) { + gpr_log(GPR_DEBUG, "%s is not available on %s", + settings_map[j].channel_arg_name, + is_client ? "clients" : "servers"); + } else { + int value = grpc_channel_arg_get_integer( + &channel_args->args[i], settings_map[j].integer_options); + if (value >= 0) { + push_setting(exec_ctx, t, settings_map[j].setting_id, + (uint32_t)value); + } + } + break; + } } } } } - set_write_state(t, GRPC_CHTTP2_WRITING_INACTIVE, "uncork"); - grpc_chttp2_initiate_write(exec_ctx, &t->global, false, "init"); + grpc_chttp2_initiate_write(exec_ctx, t, false, "init"); + post_benign_reclaimer(exec_ctx, t); } static void destroy_transport_locked(grpc_exec_ctx *exec_ctx, void *tp, grpc_error *error) { grpc_chttp2_transport *t = tp; t->destroying = 1; - drop_connection(exec_ctx, t, GRPC_ERROR_CREATE("Transport destroyed")); - UNREF_TRANSPORT(exec_ctx, t, "destroy"); + close_transport_locked( + exec_ctx, t, + grpc_error_set_int(GRPC_ERROR_CREATE("Transport destroyed"), + GRPC_ERROR_INT_OCCURRED_DURING_WRITE, t->write_state)); + GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "destroy"); } static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt; - grpc_combiner_execute(exec_ctx, t->executor.combiner, + grpc_combiner_execute(exec_ctx, t->combiner, grpc_closure_create(destroy_transport_locked, t), - GRPC_ERROR_NONE); -} - -/** block grpc_endpoint_shutdown being called until a paired - allow_endpoint_shutdown is made */ -static void prevent_endpoint_shutdown(grpc_chttp2_transport *t) { - gpr_ref(&t->shutdown_ep_refs); -} - -static void allow_endpoint_shutdown_locked(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t) { - if (gpr_unref(&t->shutdown_ep_refs)) { - grpc_endpoint_shutdown(exec_ctx, t->ep); - } + GRPC_ERROR_NONE, false); } static void close_transport_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_error *error) { if (!t->closed) { - if (grpc_http_write_state_trace) { - gpr_log(GPR_DEBUG, "W:%p close transport", t); + if (t->write_state != GRPC_CHTTP2_WRITE_STATE_IDLE) { + if (t->close_transport_on_writes_finished == NULL) { + t->close_transport_on_writes_finished = + GRPC_ERROR_CREATE("Delayed close due to in-progress write"); + } + t->close_transport_on_writes_finished = + grpc_error_add_child(t->close_transport_on_writes_finished, error); + return; + } + if (!grpc_error_get_int(error, GRPC_ERROR_INT_GRPC_STATUS, NULL)) { + error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, + GRPC_STATUS_UNAVAILABLE); } t->closed = 1; - connectivity_state_set(exec_ctx, &t->global, GRPC_CHANNEL_SHUTDOWN, + connectivity_state_set(exec_ctx, t, GRPC_CHANNEL_SHUTDOWN, GRPC_ERROR_REF(error), "close_transport"); - allow_endpoint_shutdown_locked(exec_ctx, t); + grpc_endpoint_shutdown(exec_ctx, t->ep); /* flush writable stream list to avoid dangling references */ - grpc_chttp2_stream_global *stream_global; - grpc_chttp2_stream_writing *stream_writing; - while (grpc_chttp2_list_pop_writable_stream( - &t->global, &t->writing, &stream_global, &stream_writing)) { - GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2_writing"); + grpc_chttp2_stream *s; + while (grpc_chttp2_list_pop_writable_stream(t, &s)) { + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:close"); } + end_all_the_calls(exec_ctx, t, GRPC_ERROR_REF(error)); } GRPC_ERROR_UNREF(error); } #ifdef GRPC_STREAM_REFCOUNT_DEBUG -void grpc_chttp2_stream_ref(grpc_chttp2_stream_global *stream_global, - const char *reason) { - grpc_stream_ref(STREAM_FROM_GLOBAL(stream_global)->refcount, reason); +void grpc_chttp2_stream_ref(grpc_chttp2_stream *s, const char *reason) { + grpc_stream_ref(s->refcount, reason); } -void grpc_chttp2_stream_unref(grpc_exec_ctx *exec_ctx, - grpc_chttp2_stream_global *stream_global, +void grpc_chttp2_stream_unref(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream *s, const char *reason) { - grpc_stream_unref(exec_ctx, STREAM_FROM_GLOBAL(stream_global)->refcount, - reason); + grpc_stream_unref(exec_ctx, s->refcount, reason); } #else -void grpc_chttp2_stream_ref(grpc_chttp2_stream_global *stream_global) { - grpc_stream_ref(STREAM_FROM_GLOBAL(stream_global)->refcount); +void grpc_chttp2_stream_ref(grpc_chttp2_stream *s) { + grpc_stream_ref(s->refcount); } -void grpc_chttp2_stream_unref(grpc_exec_ctx *exec_ctx, - grpc_chttp2_stream_global *stream_global) { - grpc_stream_unref(exec_ctx, STREAM_FROM_GLOBAL(stream_global)->refcount); +void grpc_chttp2_stream_unref(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream *s) { + grpc_stream_unref(exec_ctx, s->refcount); } #endif -static void finish_init_stream_locked(grpc_exec_ctx *exec_ctx, void *sp, - grpc_error *error) { - grpc_chttp2_stream *s = sp; - grpc_chttp2_register_stream(s->t, s); - GRPC_CHTTP2_STREAM_UNREF(exec_ctx, &s->global, "init"); -} - static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_stream_refcount *refcount, const void *server_data) { @@ -509,41 +466,31 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, /* We reserve one 'active stream' that's dropped when the stream is read-closed. The others are for incoming_byte_streams that are actively reading */ - gpr_ref_init(&s->global.active_streams, 1); - GRPC_CHTTP2_STREAM_REF(&s->global, "chttp2"); + gpr_ref_init(&s->active_streams, 1); + GRPC_CHTTP2_STREAM_REF(s, "chttp2"); - grpc_chttp2_incoming_metadata_buffer_init(&s->parsing.metadata_buffer[0]); - grpc_chttp2_incoming_metadata_buffer_init(&s->parsing.metadata_buffer[1]); - grpc_chttp2_incoming_metadata_buffer_init( - &s->global.received_initial_metadata); - grpc_chttp2_incoming_metadata_buffer_init( - &s->global.received_trailing_metadata); - grpc_chttp2_data_parser_init(&s->parsing.data_parser); - gpr_slice_buffer_init(&s->writing.flow_controlled_buffer); - s->global.deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); + grpc_chttp2_incoming_metadata_buffer_init(&s->metadata_buffer[0]); + grpc_chttp2_incoming_metadata_buffer_init(&s->metadata_buffer[1]); + grpc_chttp2_data_parser_init(&s->data_parser); + grpc_slice_buffer_init(&s->flow_controlled_buffer); + s->deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); + grpc_closure_init(&s->complete_fetch, complete_fetch, s); + grpc_closure_init(&s->complete_fetch_locked, complete_fetch_locked, s); - REF_TRANSPORT(t, "stream"); + GRPC_CHTTP2_REF_TRANSPORT(t, "stream"); if (server_data) { - GPR_ASSERT(t->executor.parsing_active); - s->global.id = (uint32_t)(uintptr_t)server_data; - s->parsing.id = s->global.id; - s->global.outgoing_window = - t->global.settings[GRPC_PEER_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; - s->parsing.incoming_window = s->global.max_recv_bytes = - t->global.settings[GRPC_SENT_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; + s->id = (uint32_t)(uintptr_t)server_data; + s->outgoing_window = t->settings[GRPC_PEER_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; + s->incoming_window = s->max_recv_bytes = + t->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; *t->accepting_stream = s; - grpc_chttp2_stream_map_add(&t->parsing_stream_map, s->global.id, s); - s->global.in_stream_map = true; + grpc_chttp2_stream_map_add(&t->stream_map, s->id, s); + post_destructive_reclaimer(exec_ctx, t); } - grpc_closure_init(&s->init_stream, finish_init_stream_locked, s); - GRPC_CHTTP2_STREAM_REF(&s->global, "init"); - grpc_combiner_execute(exec_ctx, t->executor.combiner, &s->init_stream, - GRPC_ERROR_NONE); - GPR_TIMER_END("init_stream", 0); return 0; @@ -557,55 +504,39 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx, void *sp, GPR_TIMER_BEGIN("destroy_stream", 0); - GPR_ASSERT((s->global.write_closed && s->global.read_closed) || - s->global.id == 0); - GPR_ASSERT(!s->global.in_stream_map); - if (grpc_chttp2_unregister_stream(t, s) && t->global.sent_goaway) { - close_transport_locked( - exec_ctx, t, - GRPC_ERROR_CREATE("Last stream closed after sending goaway")); - } - if (!t->executor.parsing_active && s->global.id) { - GPR_ASSERT(grpc_chttp2_stream_map_find(&t->parsing_stream_map, - s->global.id) == NULL); + GPR_ASSERT((s->write_closed && s->read_closed) || s->id == 0); + if (s->id != 0) { + GPR_ASSERT(grpc_chttp2_stream_map_find(&t->stream_map, s->id) == NULL); } - while ( - (bs = grpc_chttp2_incoming_frame_queue_pop(&s->global.incoming_frames))) { + while ((bs = grpc_chttp2_incoming_frame_queue_pop(&s->incoming_frames))) { incoming_byte_stream_destroy_locked(exec_ctx, bs, GRPC_ERROR_NONE); } - grpc_chttp2_list_remove_unannounced_incoming_window_available(&t->global, - &s->global); - grpc_chttp2_list_remove_stalled_by_transport(&t->global, &s->global); - grpc_chttp2_list_remove_check_read_ops(&t->global, &s->global); + grpc_chttp2_list_remove_stalled_by_transport(t, s); for (int i = 0; i < STREAM_LIST_COUNT; i++) { if (s->included[i]) { gpr_log(GPR_ERROR, "%s stream %d still included in list %d", - t->global.is_client ? "client" : "server", s->global.id, i); + t->is_client ? "client" : "server", s->id, i); abort(); } } - GPR_ASSERT(s->global.send_initial_metadata_finished == NULL); - GPR_ASSERT(s->global.send_message_finished == NULL); - GPR_ASSERT(s->global.send_trailing_metadata_finished == NULL); - GPR_ASSERT(s->global.recv_initial_metadata_ready == NULL); - GPR_ASSERT(s->global.recv_message_ready == NULL); - GPR_ASSERT(s->global.recv_trailing_metadata_finished == NULL); - grpc_chttp2_data_parser_destroy(exec_ctx, &s->parsing.data_parser); - grpc_chttp2_incoming_metadata_buffer_destroy(&s->parsing.metadata_buffer[0]); - grpc_chttp2_incoming_metadata_buffer_destroy(&s->parsing.metadata_buffer[1]); - grpc_chttp2_incoming_metadata_buffer_destroy( - &s->global.received_initial_metadata); - grpc_chttp2_incoming_metadata_buffer_destroy( - &s->global.received_trailing_metadata); - gpr_slice_buffer_destroy(&s->writing.flow_controlled_buffer); - GRPC_ERROR_UNREF(s->global.read_closed_error); - GRPC_ERROR_UNREF(s->global.write_closed_error); - - UNREF_TRANSPORT(exec_ctx, t, "stream"); + GPR_ASSERT(s->send_initial_metadata_finished == NULL); + GPR_ASSERT(s->fetching_send_message == NULL); + GPR_ASSERT(s->send_trailing_metadata_finished == NULL); + GPR_ASSERT(s->recv_initial_metadata_ready == NULL); + GPR_ASSERT(s->recv_message_ready == NULL); + GPR_ASSERT(s->recv_trailing_metadata_finished == NULL); + grpc_chttp2_data_parser_destroy(exec_ctx, &s->data_parser); + grpc_chttp2_incoming_metadata_buffer_destroy(&s->metadata_buffer[0]); + grpc_chttp2_incoming_metadata_buffer_destroy(&s->metadata_buffer[1]); + grpc_slice_buffer_destroy(&s->flow_controlled_buffer); + GRPC_ERROR_UNREF(s->read_closed_error); + GRPC_ERROR_UNREF(s->write_closed_error); + + GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "stream"); GPR_TIMER_END("destroy_stream", 0); @@ -620,280 +551,222 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, s->destroy_stream_arg = and_free_memory; grpc_closure_init(&s->destroy_stream, destroy_stream_locked, s); - grpc_combiner_execute(exec_ctx, t->executor.combiner, &s->destroy_stream, - GRPC_ERROR_NONE); + grpc_combiner_execute(exec_ctx, t->combiner, &s->destroy_stream, + GRPC_ERROR_NONE, false); GPR_TIMER_END("destroy_stream", 0); } -grpc_chttp2_stream_parsing *grpc_chttp2_parsing_lookup_stream( - grpc_chttp2_transport_parsing *transport_parsing, uint32_t id) { - grpc_chttp2_transport *t = TRANSPORT_FROM_PARSING(transport_parsing); - grpc_chttp2_stream *s = - grpc_chttp2_stream_map_find(&t->parsing_stream_map, id); - return s ? &s->parsing : NULL; +grpc_chttp2_stream *grpc_chttp2_parsing_lookup_stream(grpc_chttp2_transport *t, + uint32_t id) { + return grpc_chttp2_stream_map_find(&t->stream_map, id); } -grpc_chttp2_stream_parsing *grpc_chttp2_parsing_accept_stream( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, - uint32_t id) { +grpc_chttp2_stream *grpc_chttp2_parsing_accept_stream(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + uint32_t id) { + if (t->channel_callback.accept_stream == NULL) { + return NULL; + } grpc_chttp2_stream *accepting; - grpc_chttp2_transport *t = TRANSPORT_FROM_PARSING(transport_parsing); GPR_ASSERT(t->accepting_stream == NULL); t->accepting_stream = &accepting; t->channel_callback.accept_stream(exec_ctx, t->channel_callback.accept_stream_user_data, &t->base, (void *)(uintptr_t)id); t->accepting_stream = NULL; - return &accepting->parsing; + return accepting; } /******************************************************************************* - * LOCK MANAGEMENT + * OUTPUT PROCESSING */ -static const char *write_state_name(grpc_chttp2_write_state state) { - switch (state) { - case GRPC_CHTTP2_WRITES_CORKED: - return "CORKED"; - case GRPC_CHTTP2_WRITING_INACTIVE: - return "INACTIVE"; - case GRPC_CHTTP2_WRITE_SCHEDULED: - return "SCHEDULED"; - case GRPC_CHTTP2_WRITING: +static const char *write_state_name(grpc_chttp2_write_state st) { + switch (st) { + case GRPC_CHTTP2_WRITE_STATE_IDLE: + return "IDLE"; + case GRPC_CHTTP2_WRITE_STATE_WRITING: return "WRITING"; - case GRPC_CHTTP2_WRITING_STALE_WITH_POLLER: - return "WRITING[p=1]"; - case GRPC_CHTTP2_WRITING_STALE_NO_POLLER: - return "WRITING[p=0]"; + case GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE: + return "WRITING+MORE"; + case GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE_AND_COVERED_BY_POLLER: + return "WRITING+MORE+COVERED"; } GPR_UNREACHABLE_CODE(return "UNKNOWN"); } -static void set_write_state(grpc_chttp2_transport *t, - grpc_chttp2_write_state state, const char *reason) { - if (grpc_http_write_state_trace) { - gpr_log(GPR_DEBUG, "W:%p %s -> %s because %s", t, - write_state_name(t->executor.write_state), write_state_name(state), - reason); +static void set_write_state(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_write_state st, const char *reason) { + GRPC_CHTTP2_IF_TRACING(gpr_log(GPR_DEBUG, "W:%p %s state %s -> %s [%s]", t, + t->is_client ? "CLIENT" : "SERVER", + write_state_name(t->write_state), + write_state_name(st), reason)); + t->write_state = st; + if (st == GRPC_CHTTP2_WRITE_STATE_IDLE && + t->close_transport_on_writes_finished != NULL) { + grpc_error *err = t->close_transport_on_writes_finished; + t->close_transport_on_writes_finished = NULL; + close_transport_locked(exec_ctx, t, err); } - t->executor.write_state = state; -} - -static void initiate_writing_locked(grpc_exec_ctx *exec_ctx, void *tp, - grpc_error *error) { - grpc_chttp2_transport *t = tp; - GPR_ASSERT(t->executor.write_state == GRPC_CHTTP2_WRITE_SCHEDULED); - start_writing(exec_ctx, t); -} - -static void initiate_read_flush_locked(grpc_exec_ctx *exec_ctx, void *tp, - grpc_error *error) { - grpc_chttp2_transport *t = tp; - t->executor.check_read_ops_scheduled = false; - check_read_ops(exec_ctx, &t->global); } -/******************************************************************************* - * OUTPUT PROCESSING - */ - void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global, + grpc_chttp2_transport *t, bool covered_by_poller, const char *reason) { GPR_TIMER_BEGIN("grpc_chttp2_initiate_write", 0); - /* Perform state checks, and transition to a scheduled state if appropriate. - If we are inactive, schedule a write chain to begin once the transport - combiner finishes any executions in its current batch (which may be - scheduled AFTER this code executes). The write chain will: - - call start_writing, which verifies (under the global lock) that there - are things that need to be written by calling - grpc_chttp2_unlocking_check_writes, and if so schedules writing_action - against the current exec_ctx, to be executed OUTSIDE of the global lock - - eventually writing_action results in grpc_chttp2_terminate_writing being - called, which re-takes the global lock, updates state, checks if we need - to do *another* write immediately, and if so loops back to - start_writing. - - Current problems: - - too much lock entry/exiting - - the writing thread can become stuck indefinitely (punt through the - workqueue periodically to fix) */ - - grpc_chttp2_transport *t = TRANSPORT_FROM_GLOBAL(transport_global); - switch (t->executor.write_state) { - case GRPC_CHTTP2_WRITES_CORKED: + switch (t->write_state) { + case GRPC_CHTTP2_WRITE_STATE_IDLE: + set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING, reason); + GRPC_CHTTP2_REF_TRANSPORT(t, "writing"); + grpc_combiner_execute_finally(exec_ctx, t->combiner, + &t->write_action_begin_locked, + GRPC_ERROR_NONE, covered_by_poller); break; - case GRPC_CHTTP2_WRITING_INACTIVE: - set_write_state(t, GRPC_CHTTP2_WRITE_SCHEDULED, reason); - REF_TRANSPORT(t, "writing"); - grpc_combiner_execute_finally(exec_ctx, t->executor.combiner, - &t->initiate_writing, GRPC_ERROR_NONE, - covered_by_poller); + case GRPC_CHTTP2_WRITE_STATE_WRITING: + set_write_state( + exec_ctx, t, + covered_by_poller + ? GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE_AND_COVERED_BY_POLLER + : GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE, + reason); break; - case GRPC_CHTTP2_WRITE_SCHEDULED: + case GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE: if (covered_by_poller) { - /* upgrade to note poller is available to cover the write */ - grpc_combiner_force_async_finally(t->executor.combiner); + set_write_state( + exec_ctx, t, + GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE_AND_COVERED_BY_POLLER, + reason); } break; - case GRPC_CHTTP2_WRITING: - set_write_state(t, - covered_by_poller ? GRPC_CHTTP2_WRITING_STALE_WITH_POLLER - : GRPC_CHTTP2_WRITING_STALE_NO_POLLER, - reason); - break; - case GRPC_CHTTP2_WRITING_STALE_WITH_POLLER: - /* nothing to do: write already requested */ - break; - case GRPC_CHTTP2_WRITING_STALE_NO_POLLER: - if (covered_by_poller) { - /* upgrade to note poller is available to cover the write */ - set_write_state(t, GRPC_CHTTP2_WRITING_STALE_WITH_POLLER, reason); - } + case GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE_AND_COVERED_BY_POLLER: break; } GPR_TIMER_END("grpc_chttp2_initiate_write", 0); } -static void start_writing(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { - GPR_TIMER_BEGIN("start_writing", 0); - GPR_ASSERT(t->executor.write_state == GRPC_CHTTP2_WRITE_SCHEDULED); - if (!t->closed && - grpc_chttp2_unlocking_check_writes(exec_ctx, &t->global, &t->writing)) { - set_write_state(t, GRPC_CHTTP2_WRITING, "start_writing"); - prevent_endpoint_shutdown(t); - grpc_exec_ctx_sched(exec_ctx, &t->writing_action, GRPC_ERROR_NONE, NULL); - } else { - if (t->closed) { - set_write_state(t, GRPC_CHTTP2_WRITING_INACTIVE, - "start_writing:transport_closed"); - } else { - set_write_state(t, GRPC_CHTTP2_WRITING_INACTIVE, - "start_writing:nothing_to_write"); - } - end_waiting_for_write(exec_ctx, t, GRPC_ERROR_NONE); - UNREF_TRANSPORT(exec_ctx, t, "writing"); +void grpc_chttp2_become_writable(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, bool covered_by_poller, + const char *reason) { + if (!t->closed && grpc_chttp2_list_add_writable_stream(t, s)) { + GRPC_CHTTP2_STREAM_REF(s, "chttp2_writing:become"); + grpc_chttp2_initiate_write(exec_ctx, t, covered_by_poller, reason); } - GPR_TIMER_END("start_writing", 0); } -void grpc_chttp2_become_writable(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, - bool covered_by_poller, const char *reason) { - if (!TRANSPORT_FROM_GLOBAL(transport_global)->closed && - grpc_chttp2_list_add_writable_stream(transport_global, stream_global)) { - GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); - grpc_chttp2_initiate_write(exec_ctx, transport_global, covered_by_poller, - reason); +static void write_action_begin_locked(grpc_exec_ctx *exec_ctx, void *gt, + grpc_error *error_ignored) { + GPR_TIMER_BEGIN("write_action_begin_locked", 0); + grpc_chttp2_transport *t = gt; + GPR_ASSERT(t->write_state != GRPC_CHTTP2_WRITE_STATE_IDLE); + if (!t->closed && grpc_chttp2_begin_write(exec_ctx, t)) { + set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING, + "begin writing"); + grpc_exec_ctx_sched(exec_ctx, &t->write_action, GRPC_ERROR_NONE, NULL); + } else { + set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_IDLE, + "begin writing nothing"); + GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "writing"); } + GPR_TIMER_END("write_action_begin_locked", 0); } -static void push_setting(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, - grpc_chttp2_setting_id id, uint32_t value) { - const grpc_chttp2_setting_parameters *sp = - &grpc_chttp2_settings_parameters[id]; - uint32_t use_value = GPR_CLAMP(value, sp->min_value, sp->max_value); - if (use_value != value) { - gpr_log(GPR_INFO, "Requested parameter %s clamped from %d to %d", sp->name, - value, use_value); - } - if (use_value != t->global.settings[GRPC_LOCAL_SETTINGS][id]) { - t->global.settings[GRPC_LOCAL_SETTINGS][id] = use_value; - t->global.dirtied_local_settings = 1; - grpc_chttp2_initiate_write(exec_ctx, &t->global, false, "push_setting"); - } +static void write_action(grpc_exec_ctx *exec_ctx, void *gt, grpc_error *error) { + grpc_chttp2_transport *t = gt; + GPR_TIMER_BEGIN("write_action", 0); + grpc_endpoint_write(exec_ctx, t->ep, &t->outbuf, &t->write_action_end); + GPR_TIMER_END("write_action", 0); } -/* error may be GRPC_ERROR_NONE if there is no error allocated yet. - In that case, use "reason" as the text for a new error. */ -static void end_waiting_for_write(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, grpc_error *error) { - grpc_chttp2_stream_global *stream_global; - while (grpc_chttp2_list_pop_closed_waiting_for_writing(&t->global, - &stream_global)) { - fail_pending_writes(exec_ctx, &t->global, stream_global, - GRPC_ERROR_REF(error)); - GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "finish_writes"); - } - GRPC_ERROR_UNREF(error); +static void write_action_end(grpc_exec_ctx *exec_ctx, void *gt, + grpc_error *error) { + grpc_chttp2_transport *t = gt; + GPR_TIMER_BEGIN("write_action_end", 0); + grpc_combiner_execute(exec_ctx, t->combiner, &t->write_action_end_locked, + GRPC_ERROR_REF(error), false); + GPR_TIMER_END("write_action_end", 0); } -static void terminate_writing_with_lock(grpc_exec_ctx *exec_ctx, void *tp, - grpc_error *error) { +static void write_action_end_locked(grpc_exec_ctx *exec_ctx, void *tp, + grpc_error *error) { GPR_TIMER_BEGIN("terminate_writing_with_lock", 0); grpc_chttp2_transport *t = tp; - allow_endpoint_shutdown_locked(exec_ctx, t); if (error != GRPC_ERROR_NONE) { - drop_connection(exec_ctx, t, GRPC_ERROR_REF(error)); + close_transport_locked(exec_ctx, t, GRPC_ERROR_REF(error)); } - grpc_chttp2_cleanup_writing(exec_ctx, &t->global, &t->writing); + if (t->sent_goaway_state == GRPC_CHTTP2_GOAWAY_SEND_SCHEDULED) { + t->sent_goaway_state = GRPC_CHTTP2_GOAWAY_SENT; + if (grpc_chttp2_stream_map_size(&t->stream_map) == 0) { + close_transport_locked(exec_ctx, t, GRPC_ERROR_CREATE("goaway sent")); + } + } - end_waiting_for_write(exec_ctx, t, GRPC_ERROR_REF(error)); + grpc_chttp2_end_write(exec_ctx, t, GRPC_ERROR_REF(error)); - switch (t->executor.write_state) { - case GRPC_CHTTP2_WRITES_CORKED: - case GRPC_CHTTP2_WRITING_INACTIVE: - case GRPC_CHTTP2_WRITE_SCHEDULED: + switch (t->write_state) { + case GRPC_CHTTP2_WRITE_STATE_IDLE: GPR_UNREACHABLE_CODE(break); - case GRPC_CHTTP2_WRITING: + case GRPC_CHTTP2_WRITE_STATE_WRITING: GPR_TIMER_MARK("state=writing", 0); - set_write_state(t, GRPC_CHTTP2_WRITING_INACTIVE, "terminate_writing"); + set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_IDLE, + "finish writing"); break; - case GRPC_CHTTP2_WRITING_STALE_WITH_POLLER: - GPR_TIMER_MARK("state=writing_stale_with_poller", 0); - set_write_state(t, GRPC_CHTTP2_WRITE_SCHEDULED, "terminate_writing"); - REF_TRANSPORT(t, "writing"); - grpc_combiner_execute_finally(exec_ctx, t->executor.combiner, - &t->initiate_writing, GRPC_ERROR_NONE, - true); - break; - case GRPC_CHTTP2_WRITING_STALE_NO_POLLER: + case GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE: GPR_TIMER_MARK("state=writing_stale_no_poller", 0); - set_write_state(t, GRPC_CHTTP2_WRITE_SCHEDULED, "terminate_writing"); - REF_TRANSPORT(t, "writing"); - grpc_combiner_execute_finally(exec_ctx, t->executor.combiner, - &t->initiate_writing, GRPC_ERROR_NONE, - false); + set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING, + "continue writing [!covered]"); + GRPC_CHTTP2_REF_TRANSPORT(t, "writing"); + grpc_combiner_execute_finally(exec_ctx, t->combiner, + &t->write_action_begin_locked, + GRPC_ERROR_NONE, false); + break; + case GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE_AND_COVERED_BY_POLLER: + GPR_TIMER_MARK("state=writing_stale_with_poller", 0); + set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING, + "continue writing [covered]"); + GRPC_CHTTP2_REF_TRANSPORT(t, "writing"); + grpc_combiner_execute_finally(exec_ctx, t->combiner, + &t->write_action_begin_locked, + GRPC_ERROR_NONE, true); break; } - UNREF_TRANSPORT(exec_ctx, t, "writing"); + GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "writing"); GPR_TIMER_END("terminate_writing_with_lock", 0); } -void grpc_chttp2_terminate_writing(grpc_exec_ctx *exec_ctx, - void *transport_writing, grpc_error *error) { - GPR_TIMER_BEGIN("grpc_chttp2_terminate_writing", 0); - grpc_chttp2_transport *t = TRANSPORT_FROM_WRITING(transport_writing); - grpc_combiner_execute(exec_ctx, t->executor.combiner, &t->terminate_writing, - GRPC_ERROR_REF(error)); - GPR_TIMER_END("grpc_chttp2_terminate_writing", 0); -} - -static void writing_action(grpc_exec_ctx *exec_ctx, void *gt, - grpc_error *error) { - grpc_chttp2_transport *t = gt; - GPR_TIMER_BEGIN("writing_action", 0); - grpc_chttp2_perform_writes(exec_ctx, &t->writing, t->ep); - GPR_TIMER_END("writing_action", 0); +static void push_setting(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_setting_id id, uint32_t value) { + const grpc_chttp2_setting_parameters *sp = + &grpc_chttp2_settings_parameters[id]; + uint32_t use_value = GPR_CLAMP(value, sp->min_value, sp->max_value); + if (use_value != value) { + gpr_log(GPR_INFO, "Requested parameter %s clamped from %d to %d", sp->name, + value, use_value); + } + if (use_value != t->settings[GRPC_LOCAL_SETTINGS][id]) { + t->settings[GRPC_LOCAL_SETTINGS][id] = use_value; + t->dirtied_local_settings = 1; + grpc_chttp2_initiate_write(exec_ctx, t, false, "push_setting"); + } } -void grpc_chttp2_add_incoming_goaway( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - uint32_t goaway_error, gpr_slice goaway_text) { - char *msg = gpr_dump_slice(goaway_text, GPR_DUMP_HEX | GPR_DUMP_ASCII); +void grpc_chttp2_add_incoming_goaway(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + uint32_t goaway_error, + grpc_slice goaway_text) { + char *msg = grpc_dump_slice(goaway_text, GPR_DUMP_HEX | GPR_DUMP_ASCII); GRPC_CHTTP2_IF_TRACING( gpr_log(GPR_DEBUG, "got goaway [%d]: %s", goaway_error, msg)); - gpr_slice_unref(goaway_text); - transport_global->seen_goaway = 1; + grpc_slice_unref(goaway_text); + t->seen_goaway = 1; /* lie: use transient failure from the transport to indicate goaway has been * received */ connectivity_state_set( - exec_ctx, transport_global, GRPC_CHANNEL_TRANSIENT_FAILURE, + exec_ctx, t, GRPC_CHANNEL_TRANSIENT_FAILURE, grpc_error_set_str( grpc_error_set_int(GRPC_ERROR_CREATE("GOAWAY received"), GRPC_ERROR_INT_HTTP2_ERROR, @@ -903,61 +776,50 @@ void grpc_chttp2_add_incoming_goaway( gpr_free(msg); } -static void maybe_start_some_streams( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global) { - grpc_chttp2_stream_global *stream_global; +static void maybe_start_some_streams(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { + grpc_chttp2_stream *s; uint32_t stream_incoming_window; /* start streams where we have free grpc_chttp2_stream ids and free * concurrency */ - while (transport_global->next_stream_id <= MAX_CLIENT_STREAM_ID && - transport_global->concurrent_stream_count < - transport_global - ->settings[GRPC_PEER_SETTINGS] - [GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS] && - grpc_chttp2_list_pop_waiting_for_concurrency(transport_global, - &stream_global)) { + while (t->next_stream_id <= MAX_CLIENT_STREAM_ID && + grpc_chttp2_stream_map_size(&t->stream_map) < + t->settings[GRPC_PEER_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS] && + grpc_chttp2_list_pop_waiting_for_concurrency(t, &s)) { /* safe since we can't (legally) be parsing this stream yet */ - grpc_chttp2_stream_parsing *stream_parsing = - &STREAM_FROM_GLOBAL(stream_global)->parsing; GRPC_CHTTP2_IF_TRACING(gpr_log( GPR_DEBUG, "HTTP:%s: Allocating new grpc_chttp2_stream %p to id %d", - transport_global->is_client ? "CLI" : "SVR", stream_global, - transport_global->next_stream_id)); + t->is_client ? "CLI" : "SVR", s, t->next_stream_id)); - GPR_ASSERT(stream_global->id == 0); - stream_global->id = stream_parsing->id = transport_global->next_stream_id; - transport_global->next_stream_id += 2; + GPR_ASSERT(s->id == 0); + s->id = t->next_stream_id; + t->next_stream_id += 2; - if (transport_global->next_stream_id >= MAX_CLIENT_STREAM_ID) { - connectivity_state_set( - exec_ctx, transport_global, GRPC_CHANNEL_TRANSIENT_FAILURE, - GRPC_ERROR_CREATE("Stream IDs exhausted"), "no_more_stream_ids"); + if (t->next_stream_id >= MAX_CLIENT_STREAM_ID) { + connectivity_state_set(exec_ctx, t, GRPC_CHANNEL_TRANSIENT_FAILURE, + GRPC_ERROR_CREATE("Stream IDs exhausted"), + "no_more_stream_ids"); } - stream_global->outgoing_window = - transport_global->settings[GRPC_PEER_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; - stream_parsing->incoming_window = stream_incoming_window = - transport_global->settings[GRPC_SENT_SETTINGS] - [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; - stream_global->max_recv_bytes = - GPR_MAX(stream_incoming_window, stream_global->max_recv_bytes); - grpc_chttp2_stream_map_add( - &TRANSPORT_FROM_GLOBAL(transport_global)->new_stream_map, - stream_global->id, STREAM_FROM_GLOBAL(stream_global)); - stream_global->in_stream_map = true; - transport_global->concurrent_stream_count++; - grpc_chttp2_become_writable(exec_ctx, transport_global, stream_global, true, - "new_stream"); + s->outgoing_window = t->settings[GRPC_PEER_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; + s->incoming_window = stream_incoming_window = + t->settings[GRPC_SENT_SETTINGS] + [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; + s->max_recv_bytes = GPR_MAX(stream_incoming_window, s->max_recv_bytes); + grpc_chttp2_stream_map_add(&t->stream_map, s->id, s); + post_destructive_reclaimer(exec_ctx, t); + grpc_chttp2_become_writable(exec_ctx, t, s, true, "new_stream"); } /* cancel out streams that will never be started */ - while (transport_global->next_stream_id >= MAX_CLIENT_STREAM_ID && - grpc_chttp2_list_pop_waiting_for_concurrency(transport_global, - &stream_global)) { - cancel_from_api(exec_ctx, transport_global, stream_global, - grpc_error_set_int( - GRPC_ERROR_CREATE("Stream IDs exhausted"), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE)); + while (t->next_stream_id >= MAX_CLIENT_STREAM_ID && + grpc_chttp2_list_pop_waiting_for_concurrency(t, &s)) { + grpc_chttp2_cancel_stream( + exec_ctx, t, s, + grpc_error_set_int(GRPC_ERROR_CREATE("Stream IDs exhausted"), + GRPC_ERROR_INT_GRPC_STATUS, + GRPC_STATUS_UNAVAILABLE)); } } @@ -969,48 +831,125 @@ static grpc_closure *add_closure_barrier(grpc_closure *closure) { return closure; } -void grpc_chttp2_complete_closure_step( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, grpc_closure **pclosure, - grpc_error *error) { +static void null_then_run_closure(grpc_exec_ctx *exec_ctx, + grpc_closure **closure, grpc_error *error) { + grpc_closure *c = *closure; + *closure = NULL; + grpc_closure_run(exec_ctx, c, error); +} + +void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + grpc_closure **pclosure, + grpc_error *error, const char *desc) { grpc_closure *closure = *pclosure; + *pclosure = NULL; if (closure == NULL) { GRPC_ERROR_UNREF(error); return; } closure->next_data.scratch -= CLOSURE_BARRIER_FIRST_REF_BIT; if (error != GRPC_ERROR_NONE) { - if (closure->error == GRPC_ERROR_NONE) { - closure->error = + if (closure->error_data.error == GRPC_ERROR_NONE) { + closure->error_data.error = GRPC_ERROR_CREATE("Error in HTTP transport completing operation"); - closure->error = grpc_error_set_str( - closure->error, GRPC_ERROR_STR_TARGET_ADDRESS, - TRANSPORT_FROM_GLOBAL(transport_global)->peer_string); + closure->error_data.error = + grpc_error_set_str(closure->error_data.error, + GRPC_ERROR_STR_TARGET_ADDRESS, t->peer_string); } - closure->error = grpc_error_add_child(closure->error, error); + closure->error_data.error = + grpc_error_add_child(closure->error_data.error, error); } if (closure->next_data.scratch < CLOSURE_BARRIER_FIRST_REF_BIT) { if (closure->next_data.scratch & CLOSURE_BARRIER_STATS_BIT) { - grpc_transport_move_stats(&stream_global->stats, - stream_global->collecting_stats); - stream_global->collecting_stats = NULL; + grpc_transport_move_stats(&s->stats, s->collecting_stats); + s->collecting_stats = NULL; } - grpc_exec_ctx_sched(exec_ctx, closure, closure->error, NULL); + grpc_closure_run(exec_ctx, closure, closure->error_data.error); } - *pclosure = NULL; } -static int contains_non_ok_status( - grpc_chttp2_transport_global *transport_global, - grpc_metadata_batch *batch) { +static bool contains_non_ok_status(grpc_metadata_batch *batch) { grpc_linked_mdelem *l; for (l = batch->list.head; l; l = l->next) { if (l->md->key == GRPC_MDSTR_GRPC_STATUS && l->md != GRPC_MDELEM_GRPC_STATUS_0) { - return 1; + return true; } } - return 0; + return false; +} + +static void add_fetched_slice_locked(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { + s->fetched_send_message_length += + (uint32_t)GRPC_SLICE_LENGTH(s->fetching_slice); + grpc_slice_buffer_add(&s->flow_controlled_buffer, s->fetching_slice); + if (s->id != 0) { + grpc_chttp2_become_writable(exec_ctx, t, s, true, "op.send_message"); + } +} + +static void continue_fetching_send_locked(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { + for (;;) { + if (s->fetching_send_message == NULL) { + /* Stream was cancelled before message fetch completed */ + abort(); /* TODO(ctiller): what cleanup here? */ + return; /* early out */ + } + if (s->fetched_send_message_length == s->fetching_send_message->length) { + int64_t notify_offset = s->next_message_end_offset; + if (notify_offset <= s->flow_controlled_bytes_written) { + grpc_chttp2_complete_closure_step( + exec_ctx, t, s, &s->fetching_send_message_finished, GRPC_ERROR_NONE, + "fetching_send_message_finished"); + } else { + grpc_chttp2_write_cb *cb = t->write_cb_pool; + if (cb == NULL) { + cb = gpr_malloc(sizeof(*cb)); + } else { + t->write_cb_pool = cb->next; + } + cb->call_at_byte = notify_offset; + cb->closure = s->fetching_send_message_finished; + s->fetching_send_message_finished = NULL; + cb->next = s->on_write_finished_cbs; + s->on_write_finished_cbs = cb; + } + s->fetching_send_message = NULL; + return; /* early out */ + } else if (grpc_byte_stream_next(exec_ctx, s->fetching_send_message, + &s->fetching_slice, UINT32_MAX, + &s->complete_fetch)) { + add_fetched_slice_locked(exec_ctx, t, s); + } + } +} + +static void complete_fetch_locked(grpc_exec_ctx *exec_ctx, void *gs, + grpc_error *error) { + grpc_chttp2_stream *s = gs; + grpc_chttp2_transport *t = s->t; + if (error == GRPC_ERROR_NONE) { + add_fetched_slice_locked(exec_ctx, t, s); + continue_fetching_send_locked(exec_ctx, t, s); + } else { + /* TODO(ctiller): what to do here */ + abort(); + } +} + +static void complete_fetch(grpc_exec_ctx *exec_ctx, void *gs, + grpc_error *error) { + grpc_chttp2_stream *s = gs; + grpc_chttp2_transport *t = s->t; + grpc_combiner_execute(exec_ctx, t->combiner, &s->complete_fetch_locked, + GRPC_ERROR_REF(error), + s->complete_fetch_covered_by_poller); } static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {} @@ -1022,12 +961,11 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, grpc_transport_stream_op *op = stream_op; grpc_chttp2_transport *t = op->transport_private.args[0]; grpc_chttp2_stream *s = op->transport_private.args[1]; - grpc_chttp2_transport_global *transport_global = &t->global; - grpc_chttp2_stream_global *stream_global = &s->global; if (grpc_http_trace) { char *str = grpc_transport_stream_op_string(op); - gpr_log(GPR_DEBUG, "perform_stream_op_locked: %s", str); + gpr_log(GPR_DEBUG, "perform_stream_op_locked: %s; on_complete = %p", str, + op->on_complete); gpr_free(str); } @@ -1035,45 +973,42 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, if (on_complete == NULL) { on_complete = grpc_closure_create(do_nothing, NULL); } + /* use final_data as a barrier until enqueue time; the inital counter is dropped at the end of this function */ on_complete->next_data.scratch = CLOSURE_BARRIER_FIRST_REF_BIT; - on_complete->error = GRPC_ERROR_NONE; + on_complete->error_data.error = GRPC_ERROR_NONE; if (op->collect_stats != NULL) { - GPR_ASSERT(stream_global->collecting_stats == NULL); - stream_global->collecting_stats = op->collect_stats; + GPR_ASSERT(s->collecting_stats == NULL); + s->collecting_stats = op->collect_stats; on_complete->next_data.scratch |= CLOSURE_BARRIER_STATS_BIT; } if (op->cancel_error != GRPC_ERROR_NONE) { - cancel_from_api(exec_ctx, transport_global, stream_global, - GRPC_ERROR_REF(op->cancel_error)); + grpc_chttp2_cancel_stream(exec_ctx, t, s, GRPC_ERROR_REF(op->cancel_error)); } if (op->close_error != GRPC_ERROR_NONE) { - close_from_api(exec_ctx, transport_global, stream_global, - GRPC_ERROR_REF(op->close_error)); + close_from_api(exec_ctx, t, s, GRPC_ERROR_REF(op->close_error)); } if (op->send_initial_metadata != NULL) { - GPR_ASSERT(stream_global->send_initial_metadata_finished == NULL); - stream_global->send_initial_metadata_finished = - add_closure_barrier(on_complete); - stream_global->send_initial_metadata = op->send_initial_metadata; + GPR_ASSERT(s->send_initial_metadata_finished == NULL); + s->send_initial_metadata_finished = add_closure_barrier(on_complete); + s->send_initial_metadata = op->send_initial_metadata; const size_t metadata_size = grpc_metadata_batch_size(op->send_initial_metadata); const size_t metadata_peer_limit = - transport_global->settings[GRPC_PEER_SETTINGS] - [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; - if (transport_global->is_client) { - stream_global->deadline = - gpr_time_min(stream_global->deadline, - stream_global->send_initial_metadata->deadline); + t->settings[GRPC_PEER_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; + if (t->is_client) { + s->deadline = + gpr_time_min(s->deadline, s->send_initial_metadata->deadline); } if (metadata_size > metadata_peer_limit) { - cancel_from_api( - exec_ctx, transport_global, stream_global, + grpc_chttp2_cancel_stream( + exec_ctx, t, s, grpc_error_set_int( grpc_error_set_int( grpc_error_set_int( @@ -1083,64 +1018,83 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, GRPC_ERROR_INT_LIMIT, (intptr_t)metadata_peer_limit), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED)); } else { - if (contains_non_ok_status(transport_global, op->send_initial_metadata)) { - stream_global->seen_error = true; - grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global, - stream_global); + if (contains_non_ok_status(op->send_initial_metadata)) { + s->seen_error = true; } - if (!stream_global->write_closed) { - if (transport_global->is_client) { - GPR_ASSERT(stream_global->id == 0); - grpc_chttp2_list_add_waiting_for_concurrency(transport_global, - stream_global); - maybe_start_some_streams(exec_ctx, transport_global); + if (!s->write_closed) { + if (t->is_client) { + if (!t->closed) { + GPR_ASSERT(s->id == 0); + grpc_chttp2_list_add_waiting_for_concurrency(t, s); + maybe_start_some_streams(exec_ctx, t); + } else { + grpc_chttp2_cancel_stream(exec_ctx, t, s, + GRPC_ERROR_CREATE("Transport closed")); + } } else { - GPR_ASSERT(stream_global->id != 0); - grpc_chttp2_become_writable(exec_ctx, transport_global, stream_global, - true, "op.send_initial_metadata"); + GPR_ASSERT(s->id != 0); + grpc_chttp2_become_writable(exec_ctx, t, s, true, + "op.send_initial_metadata"); } } else { - stream_global->send_trailing_metadata = NULL; + s->send_trailing_metadata = NULL; grpc_chttp2_complete_closure_step( - exec_ctx, transport_global, stream_global, - &stream_global->send_initial_metadata_finished, + exec_ctx, t, s, &s->send_initial_metadata_finished, GRPC_ERROR_CREATE( - "Attempt to send initial metadata after stream was closed")); + "Attempt to send initial metadata after stream was closed"), + "send_initial_metadata_finished"); } } } if (op->send_message != NULL) { - GPR_ASSERT(stream_global->send_message_finished == NULL); - GPR_ASSERT(stream_global->send_message == NULL); - stream_global->send_message_finished = add_closure_barrier(on_complete); - if (stream_global->write_closed) { + s->fetching_send_message_finished = add_closure_barrier(op->on_complete); + if (s->write_closed) { grpc_chttp2_complete_closure_step( - exec_ctx, transport_global, stream_global, - &stream_global->send_message_finished, - GRPC_ERROR_CREATE("Attempt to send message after stream was closed")); + exec_ctx, t, s, &s->fetching_send_message_finished, + GRPC_ERROR_CREATE("Attempt to send message after stream was closed"), + "fetching_send_message_finished"); } else { - stream_global->send_message = op->send_message; - if (stream_global->id != 0) { - grpc_chttp2_become_writable(exec_ctx, transport_global, stream_global, - true, "op.send_message"); + GPR_ASSERT(s->fetching_send_message == NULL); + uint8_t *frame_hdr = + grpc_slice_buffer_tiny_add(&s->flow_controlled_buffer, 5); + uint32_t flags = op->send_message->flags; + frame_hdr[0] = (flags & GRPC_WRITE_INTERNAL_COMPRESS) != 0; + size_t len = op->send_message->length; + frame_hdr[1] = (uint8_t)(len >> 24); + frame_hdr[2] = (uint8_t)(len >> 16); + frame_hdr[3] = (uint8_t)(len >> 8); + frame_hdr[4] = (uint8_t)(len); + s->fetching_send_message = op->send_message; + s->fetched_send_message_length = 0; + s->next_message_end_offset = s->flow_controlled_bytes_written + + (int64_t)s->flow_controlled_buffer.length + + (int64_t)len; + s->complete_fetch_covered_by_poller = op->covered_by_poller; + if (flags & GRPC_WRITE_BUFFER_HINT) { + /* allow up to 64kb to be buffered */ + /* TODO(ctiller): make this configurable */ + s->next_message_end_offset -= 65536; + } + continue_fetching_send_locked(exec_ctx, t, s); + if (s->id != 0) { + grpc_chttp2_become_writable(exec_ctx, t, s, true, "op.send_message"); } } } if (op->send_trailing_metadata != NULL) { - GPR_ASSERT(stream_global->send_trailing_metadata_finished == NULL); - stream_global->send_trailing_metadata_finished = - add_closure_barrier(on_complete); - stream_global->send_trailing_metadata = op->send_trailing_metadata; + GPR_ASSERT(s->send_trailing_metadata_finished == NULL); + s->send_trailing_metadata_finished = add_closure_barrier(on_complete); + s->send_trailing_metadata = op->send_trailing_metadata; const size_t metadata_size = grpc_metadata_batch_size(op->send_trailing_metadata); const size_t metadata_peer_limit = - transport_global->settings[GRPC_PEER_SETTINGS] - [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; + t->settings[GRPC_PEER_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; if (metadata_size > metadata_peer_limit) { - cancel_from_api( - exec_ctx, transport_global, stream_global, + grpc_chttp2_cancel_stream( + exec_ctx, t, s, grpc_error_set_int( grpc_error_set_int( grpc_error_set_int( @@ -1150,69 +1104,59 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, GRPC_ERROR_INT_LIMIT, (intptr_t)metadata_peer_limit), GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED)); } else { - if (contains_non_ok_status(transport_global, - op->send_trailing_metadata)) { - stream_global->seen_error = true; - grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global, - stream_global); + if (contains_non_ok_status(op->send_trailing_metadata)) { + s->seen_error = true; } - if (stream_global->write_closed) { - stream_global->send_trailing_metadata = NULL; + if (s->write_closed) { + s->send_trailing_metadata = NULL; grpc_chttp2_complete_closure_step( - exec_ctx, transport_global, stream_global, - &stream_global->send_trailing_metadata_finished, + exec_ctx, t, s, &s->send_trailing_metadata_finished, grpc_metadata_batch_is_empty(op->send_trailing_metadata) ? GRPC_ERROR_NONE : GRPC_ERROR_CREATE("Attempt to send trailing metadata after " - "stream was closed")); - } else if (stream_global->id != 0) { + "stream was closed"), + "send_trailing_metadata_finished"); + } else if (s->id != 0) { /* TODO(ctiller): check if there's flow control for any outstanding bytes before going writable */ - grpc_chttp2_become_writable(exec_ctx, transport_global, stream_global, - true, "op.send_trailing_metadata"); + grpc_chttp2_become_writable(exec_ctx, t, s, true, + "op.send_trailing_metadata"); } } } if (op->recv_initial_metadata != NULL) { - GPR_ASSERT(stream_global->recv_initial_metadata_ready == NULL); - stream_global->recv_initial_metadata_ready = - op->recv_initial_metadata_ready; - stream_global->recv_initial_metadata = op->recv_initial_metadata; - grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global, - stream_global); + GPR_ASSERT(s->recv_initial_metadata_ready == NULL); + s->recv_initial_metadata_ready = op->recv_initial_metadata_ready; + s->recv_initial_metadata = op->recv_initial_metadata; + grpc_chttp2_maybe_complete_recv_initial_metadata(exec_ctx, t, s); } if (op->recv_message != NULL) { - GPR_ASSERT(stream_global->recv_message_ready == NULL); - stream_global->recv_message_ready = op->recv_message_ready; - stream_global->recv_message = op->recv_message; - if (stream_global->id != 0 && - (stream_global->incoming_frames.head == NULL || - stream_global->incoming_frames.head->is_tail)) { - incoming_byte_stream_update_flow_control( - exec_ctx, transport_global, stream_global, - transport_global->stream_lookahead, 0); + GPR_ASSERT(s->recv_message_ready == NULL); + s->recv_message_ready = op->recv_message_ready; + s->recv_message = op->recv_message; + if (s->id != 0 && + (s->incoming_frames.head == NULL || s->incoming_frames.head->is_tail)) { + incoming_byte_stream_update_flow_control(exec_ctx, t, s, + t->stream_lookahead, 0); } - grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global, - stream_global); + grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s); } if (op->recv_trailing_metadata != NULL) { - GPR_ASSERT(stream_global->recv_trailing_metadata_finished == NULL); - stream_global->recv_trailing_metadata_finished = - add_closure_barrier(on_complete); - stream_global->recv_trailing_metadata = op->recv_trailing_metadata; - stream_global->final_metadata_requested = true; - grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global, - stream_global); + GPR_ASSERT(s->recv_trailing_metadata_finished == NULL); + s->recv_trailing_metadata_finished = add_closure_barrier(on_complete); + s->recv_trailing_metadata = op->recv_trailing_metadata; + s->final_metadata_requested = true; + grpc_chttp2_maybe_complete_recv_trailing_metadata(exec_ctx, t, s); } - grpc_chttp2_complete_closure_step(exec_ctx, transport_global, stream_global, - &on_complete, GRPC_ERROR_NONE); + grpc_chttp2_complete_closure_step(exec_ctx, t, s, &on_complete, + GRPC_ERROR_NONE, "op->on_complete"); GPR_TIMER_END("perform_stream_op_locked", 0); - GRPC_CHTTP2_STREAM_UNREF(exec_ctx, &s->global, "perform_stream_op"); + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "perform_stream_op"); } static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, @@ -1220,70 +1164,68 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, GPR_TIMER_BEGIN("perform_stream_op", 0); grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt; grpc_chttp2_stream *s = (grpc_chttp2_stream *)gs; + + if (grpc_http_trace) { + char *str = grpc_transport_stream_op_string(op); + gpr_log(GPR_DEBUG, "perform_stream_op[s=%p/%d]: %s", s, s->id, str); + gpr_free(str); + } + grpc_closure_init(&op->transport_private.closure, perform_stream_op_locked, op); op->transport_private.args[0] = gt; op->transport_private.args[1] = gs; - GRPC_CHTTP2_STREAM_REF(&s->global, "perform_stream_op"); - grpc_combiner_execute(exec_ctx, t->executor.combiner, - &op->transport_private.closure, GRPC_ERROR_NONE); + GRPC_CHTTP2_STREAM_REF(s, "perform_stream_op"); + grpc_combiner_execute(exec_ctx, t->combiner, &op->transport_private.closure, + GRPC_ERROR_NONE, op->covered_by_poller); GPR_TIMER_END("perform_stream_op", 0); } static void send_ping_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_closure *on_recv) { grpc_chttp2_outstanding_ping *p = gpr_malloc(sizeof(*p)); - p->next = &t->global.pings; + p->next = &t->pings; p->prev = p->next->prev; p->prev->next = p->next->prev = p; - p->id[0] = (uint8_t)((t->global.ping_counter >> 56) & 0xff); - p->id[1] = (uint8_t)((t->global.ping_counter >> 48) & 0xff); - p->id[2] = (uint8_t)((t->global.ping_counter >> 40) & 0xff); - p->id[3] = (uint8_t)((t->global.ping_counter >> 32) & 0xff); - p->id[4] = (uint8_t)((t->global.ping_counter >> 24) & 0xff); - p->id[5] = (uint8_t)((t->global.ping_counter >> 16) & 0xff); - p->id[6] = (uint8_t)((t->global.ping_counter >> 8) & 0xff); - p->id[7] = (uint8_t)(t->global.ping_counter & 0xff); + p->id[0] = (uint8_t)((t->ping_counter >> 56) & 0xff); + p->id[1] = (uint8_t)((t->ping_counter >> 48) & 0xff); + p->id[2] = (uint8_t)((t->ping_counter >> 40) & 0xff); + p->id[3] = (uint8_t)((t->ping_counter >> 32) & 0xff); + p->id[4] = (uint8_t)((t->ping_counter >> 24) & 0xff); + p->id[5] = (uint8_t)((t->ping_counter >> 16) & 0xff); + p->id[6] = (uint8_t)((t->ping_counter >> 8) & 0xff); + p->id[7] = (uint8_t)(t->ping_counter & 0xff); + t->ping_counter++; p->on_recv = on_recv; - gpr_slice_buffer_add(&t->global.qbuf, grpc_chttp2_ping_create(0, p->id)); - grpc_chttp2_initiate_write(exec_ctx, &t->global, true, "send_ping"); + grpc_slice_buffer_add(&t->qbuf, grpc_chttp2_ping_create(0, p->id)); + grpc_chttp2_initiate_write(exec_ctx, t, true, "send_ping"); } -typedef struct ack_ping_args { - grpc_closure closure; - grpc_chttp2_transport *t; - uint8_t opaque_8bytes[8]; -} ack_ping_args; - -static void ack_ping_locked(grpc_exec_ctx *exec_ctx, void *a, - grpc_error *error_ignored) { - ack_ping_args *args = a; +void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + const uint8_t *opaque_8bytes) { grpc_chttp2_outstanding_ping *ping; - grpc_chttp2_transport_global *transport_global = &args->t->global; - for (ping = transport_global->pings.next; ping != &transport_global->pings; - ping = ping->next) { - if (0 == memcmp(args->opaque_8bytes, ping->id, 8)) { + for (ping = t->pings.next; ping != &t->pings; ping = ping->next) { + if (0 == memcmp(opaque_8bytes, ping->id, 8)) { grpc_exec_ctx_sched(exec_ctx, ping->on_recv, GRPC_ERROR_NONE, NULL); ping->next->prev = ping->prev; ping->prev->next = ping->next; gpr_free(ping); - break; + return; } } - UNREF_TRANSPORT(exec_ctx, args->t, "ack_ping"); - gpr_free(args); + char *msg = gpr_dump((const char *)opaque_8bytes, 8, GPR_DUMP_HEX); + char *from = grpc_endpoint_get_peer(t->ep); + gpr_log(GPR_DEBUG, "Unknown ping response from %s: %s", from, msg); + gpr_free(from); + gpr_free(msg); } -void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_parsing *transport_parsing, - const uint8_t *opaque_8bytes) { - ack_ping_args *args = gpr_malloc(sizeof(*args)); - args->t = TRANSPORT_FROM_PARSING(transport_parsing); - memcpy(args->opaque_8bytes, opaque_8bytes, sizeof(args->opaque_8bytes)); - grpc_closure_init(&args->closure, ack_ping_locked, args); - REF_TRANSPORT(args->t, "ack_ping"); - grpc_combiner_execute(exec_ctx, args->t->executor.combiner, &args->closure, - GRPC_ERROR_NONE); +static void send_goaway(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_error_code error, grpc_slice data) { + t->sent_goaway_state = GRPC_CHTTP2_GOAWAY_SEND_SCHEDULED; + grpc_chttp2_goaway_append(t->last_new_stream_id, (uint32_t)error, data, + &t->qbuf); + grpc_chttp2_initiate_write(exec_ctx, t, false, "goaway_sent"); } static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx, @@ -1293,15 +1235,6 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t = op->transport_private.args[0]; grpc_error *close_transport = op->disconnect_with_error; - /* If there's a set_accept_stream ensure that we're not parsing - to avoid changing things out from underneath */ - if (t->executor.parsing_active && op->set_accept_stream) { - GPR_ASSERT(t->post_parsing_op == NULL); - t->post_parsing_op = gpr_malloc(sizeof(*op)); - memcpy(t->post_parsing_op, op, sizeof(*op)); - return; - } - if (op->on_connectivity_state_change != NULL) { grpc_connectivity_state_notify_on_state_change( exec_ctx, &t->channel_callback.state_tracker, op->connectivity_state, @@ -1309,15 +1242,9 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx, } if (op->send_goaway) { - t->global.sent_goaway = 1; - grpc_chttp2_goaway_append( - t->global.last_incoming_stream_id, - (uint32_t)grpc_chttp2_grpc_status_to_http2_error(op->goaway_status), - gpr_slice_ref(*op->goaway_message), &t->global.qbuf); - close_transport = grpc_chttp2_has_streams(t) - ? GRPC_ERROR_NONE - : GRPC_ERROR_CREATE("GOAWAY sent"); - grpc_chttp2_initiate_write(exec_ctx, &t->global, false, "goaway_sent"); + send_goaway(exec_ctx, t, + grpc_chttp2_grpc_status_to_http2_error(op->goaway_status), + grpc_slice_ref(*op->goaway_message)); } if (op->set_accept_stream) { @@ -1342,154 +1269,131 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx, close_transport_locked(exec_ctx, t, close_transport); } - grpc_exec_ctx_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL); + grpc_closure_run(exec_ctx, op->on_consumed, GRPC_ERROR_NONE); - UNREF_TRANSPORT(exec_ctx, t, "transport_op"); + GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "transport_op"); } static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_transport_op *op) { grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt; + char *msg = grpc_transport_op_string(op); + gpr_free(msg); op->transport_private.args[0] = gt; grpc_closure_init(&op->transport_private.closure, perform_transport_op_locked, op); - REF_TRANSPORT(t, "transport_op"); - grpc_combiner_execute(exec_ctx, t->executor.combiner, - &op->transport_private.closure, GRPC_ERROR_NONE); + GRPC_CHTTP2_REF_TRANSPORT(t, "transport_op"); + grpc_combiner_execute(exec_ctx, t->combiner, &op->transport_private.closure, + GRPC_ERROR_NONE, false); } /******************************************************************************* * INPUT PROCESSING - GENERAL */ -static void check_read_ops(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global) { - GPR_TIMER_BEGIN("check_read_ops", 0); - grpc_chttp2_stream_global *stream_global; +void grpc_chttp2_maybe_complete_recv_initial_metadata(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { grpc_byte_stream *bs; - while ( - grpc_chttp2_list_pop_check_read_ops(transport_global, &stream_global)) { - if (stream_global->recv_initial_metadata_ready != NULL && - stream_global->published_initial_metadata) { - if (stream_global->seen_error) { - while ((bs = grpc_chttp2_incoming_frame_queue_pop( - &stream_global->incoming_frames)) != NULL) { - incoming_byte_stream_destroy_locked(exec_ctx, bs, GRPC_ERROR_NONE); - } - if (stream_global->exceeded_metadata_size) { - cancel_from_api( - exec_ctx, transport_global, stream_global, - grpc_error_set_int( - GRPC_ERROR_CREATE( - "received initial metadata size exceeds limit"), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED)); - } + if (s->recv_initial_metadata_ready != NULL && + s->published_metadata[0] != GRPC_METADATA_NOT_PUBLISHED) { + if (s->seen_error) { + while ((bs = grpc_chttp2_incoming_frame_queue_pop(&s->incoming_frames)) != + NULL) { + incoming_byte_stream_destroy_locked(exec_ctx, bs, GRPC_ERROR_NONE); } - grpc_chttp2_incoming_metadata_buffer_publish( - &stream_global->received_initial_metadata, - stream_global->recv_initial_metadata); - grpc_exec_ctx_sched(exec_ctx, stream_global->recv_initial_metadata_ready, - GRPC_ERROR_NONE, NULL); - stream_global->recv_initial_metadata_ready = NULL; } - if (stream_global->recv_message_ready != NULL) { - while (stream_global->final_metadata_requested && - stream_global->seen_error && - (bs = grpc_chttp2_incoming_frame_queue_pop( - &stream_global->incoming_frames)) != NULL) { + grpc_chttp2_incoming_metadata_buffer_publish(&s->metadata_buffer[0], + s->recv_initial_metadata); + null_then_run_closure(exec_ctx, &s->recv_initial_metadata_ready, + GRPC_ERROR_NONE); + } +} + +void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { + grpc_byte_stream *bs; + if (s->recv_message_ready != NULL) { + while (s->final_metadata_requested && s->seen_error && + (bs = grpc_chttp2_incoming_frame_queue_pop(&s->incoming_frames)) != + NULL) { + incoming_byte_stream_destroy_locked(exec_ctx, bs, GRPC_ERROR_NONE); + } + if (s->incoming_frames.head != NULL) { + *s->recv_message = + grpc_chttp2_incoming_frame_queue_pop(&s->incoming_frames); + GPR_ASSERT(*s->recv_message != NULL); + null_then_run_closure(exec_ctx, &s->recv_message_ready, GRPC_ERROR_NONE); + } else if (s->published_metadata[1] != GRPC_METADATA_NOT_PUBLISHED) { + *s->recv_message = NULL; + null_then_run_closure(exec_ctx, &s->recv_message_ready, GRPC_ERROR_NONE); + } + } +} + +void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { + grpc_byte_stream *bs; + grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s); + if (s->recv_trailing_metadata_finished != NULL && s->read_closed && + s->write_closed) { + if (s->seen_error) { + while ((bs = grpc_chttp2_incoming_frame_queue_pop(&s->incoming_frames)) != + NULL) { incoming_byte_stream_destroy_locked(exec_ctx, bs, GRPC_ERROR_NONE); } - if (stream_global->incoming_frames.head != NULL) { - *stream_global->recv_message = grpc_chttp2_incoming_frame_queue_pop( - &stream_global->incoming_frames); - GPR_ASSERT(*stream_global->recv_message != NULL); - grpc_exec_ctx_sched(exec_ctx, stream_global->recv_message_ready, - GRPC_ERROR_NONE, NULL); - stream_global->recv_message_ready = NULL; - } else if (stream_global->published_trailing_metadata) { - *stream_global->recv_message = NULL; - grpc_exec_ctx_sched(exec_ctx, stream_global->recv_message_ready, - GRPC_ERROR_NONE, NULL); - stream_global->recv_message_ready = NULL; - } } - if (stream_global->recv_trailing_metadata_finished != NULL && - stream_global->read_closed && stream_global->write_closed) { - if (stream_global->seen_error) { - while ((bs = grpc_chttp2_incoming_frame_queue_pop( - &stream_global->incoming_frames)) != NULL) { - incoming_byte_stream_destroy_locked(exec_ctx, bs, GRPC_ERROR_NONE); - } - if (stream_global->exceeded_metadata_size) { - cancel_from_api( - exec_ctx, transport_global, stream_global, - grpc_error_set_int( - GRPC_ERROR_CREATE( - "received trailing metadata size exceeds limit"), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED)); - } - } - if (stream_global->all_incoming_byte_streams_finished) { - grpc_chttp2_incoming_metadata_buffer_publish( - &stream_global->received_trailing_metadata, - stream_global->recv_trailing_metadata); - grpc_chttp2_complete_closure_step( - exec_ctx, transport_global, stream_global, - &stream_global->recv_trailing_metadata_finished, GRPC_ERROR_NONE); - } + if (s->all_incoming_byte_streams_finished && + s->recv_trailing_metadata_finished != NULL) { + grpc_chttp2_incoming_metadata_buffer_publish(&s->metadata_buffer[1], + s->recv_trailing_metadata); + grpc_chttp2_complete_closure_step( + exec_ctx, t, s, &s->recv_trailing_metadata_finished, GRPC_ERROR_NONE, + "recv_trailing_metadata_finished"); } } - GPR_TIMER_END("check_read_ops", 0); } -static void decrement_active_streams_locked( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) { - if ((stream_global->all_incoming_byte_streams_finished = - gpr_unref(&stream_global->active_streams))) { - grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global, - stream_global); +static void decrement_active_streams_locked(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { + if ((s->all_incoming_byte_streams_finished = gpr_unref(&s->active_streams))) { + grpc_chttp2_maybe_complete_recv_trailing_metadata(exec_ctx, t, s); } } static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, uint32_t id, grpc_error *error) { - size_t new_stream_count; - grpc_chttp2_stream *s = - grpc_chttp2_stream_map_delete(&t->parsing_stream_map, id); - if (!s) { - s = grpc_chttp2_stream_map_delete(&t->new_stream_map, id); - } + grpc_chttp2_stream *s = grpc_chttp2_stream_map_delete(&t->stream_map, id); GPR_ASSERT(s); - s->global.in_stream_map = false; - if (t->parsing.incoming_stream == &s->parsing) { - t->parsing.incoming_stream = NULL; - grpc_chttp2_parsing_become_skip_parser(exec_ctx, &t->parsing); + if (t->incoming_stream == s) { + t->incoming_stream = NULL; + grpc_chttp2_parsing_become_skip_parser(exec_ctx, t); } - if (s->parsing.data_parser.parsing_frame != NULL) { + if (s->data_parser.parsing_frame != NULL) { grpc_chttp2_incoming_byte_stream_finished( - exec_ctx, s->parsing.data_parser.parsing_frame, GRPC_ERROR_REF(error), - 0); - s->parsing.data_parser.parsing_frame = NULL; + exec_ctx, s->data_parser.parsing_frame, GRPC_ERROR_REF(error)); + s->data_parser.parsing_frame = NULL; } - if (grpc_chttp2_unregister_stream(t, s) && t->global.sent_goaway) { - close_transport_locked( - exec_ctx, t, GRPC_ERROR_CREATE_REFERENCING( - "Last stream closed after sending GOAWAY", &error, 1)); + if (grpc_chttp2_stream_map_size(&t->stream_map) == 0) { + post_benign_reclaimer(exec_ctx, t); + if (t->sent_goaway_state == GRPC_CHTTP2_GOAWAY_SENT) { + close_transport_locked( + exec_ctx, t, + GRPC_ERROR_CREATE_REFERENCING( + "Last stream closed after sending GOAWAY", &error, 1)); + } } - if (grpc_chttp2_list_remove_writable_stream(&t->global, &s->global)) { - GRPC_CHTTP2_STREAM_UNREF(exec_ctx, &s->global, "chttp2_writing"); + if (grpc_chttp2_list_remove_writable_stream(t, s)) { + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:remove_stream"); } - new_stream_count = grpc_chttp2_stream_map_size(&t->parsing_stream_map) + - grpc_chttp2_stream_map_size(&t->new_stream_map); - GPR_ASSERT(new_stream_count <= UINT32_MAX); - if (new_stream_count != t->global.concurrent_stream_count) { - t->global.concurrent_stream_count = (uint32_t)new_stream_count; - maybe_start_some_streams(exec_ctx, &t->global); - } GRPC_ERROR_UNREF(error); + + maybe_start_some_streams(exec_ctx, t); } static void status_codes_from_error(grpc_error *error, gpr_timespec deadline, @@ -1519,23 +1423,20 @@ static void status_codes_from_error(grpc_error *error, gpr_timespec deadline, } } -static void cancel_from_api(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, - grpc_error *due_to_error) { - if (!stream_global->read_closed || !stream_global->write_closed) { +void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, grpc_chttp2_stream *s, + grpc_error *due_to_error) { + if (!s->read_closed || !s->write_closed) { grpc_status_code grpc_status; grpc_chttp2_error_code http_error; - status_codes_from_error(due_to_error, stream_global->deadline, &http_error, + status_codes_from_error(due_to_error, s->deadline, &http_error, &grpc_status); - if (stream_global->id != 0) { - gpr_slice_buffer_add( - &transport_global->qbuf, - grpc_chttp2_rst_stream_create(stream_global->id, (uint32_t)http_error, - &stream_global->stats.outgoing)); - grpc_chttp2_initiate_write(exec_ctx, transport_global, false, - "rst_stream"); + if (s->id != 0) { + grpc_slice_buffer_add( + &t->qbuf, grpc_chttp2_rst_stream_create(s->id, (uint32_t)http_error, + &s->stats.outgoing)); + grpc_chttp2_initiate_write(exec_ctx, t, false, "rst_stream"); } const char *msg = @@ -1545,28 +1446,22 @@ static void cancel_from_api(grpc_exec_ctx *exec_ctx, free_msg = true; msg = grpc_error_string(due_to_error); } - gpr_slice msg_slice = gpr_slice_from_copied_string(msg); - grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global, - grpc_status, &msg_slice); + grpc_slice msg_slice = grpc_slice_from_copied_string(msg); + grpc_chttp2_fake_status(exec_ctx, t, s, grpc_status, &msg_slice); if (free_msg) grpc_error_free_string(msg); } - if (due_to_error != GRPC_ERROR_NONE && !stream_global->seen_error) { - stream_global->seen_error = true; - grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global, - stream_global); + if (due_to_error != GRPC_ERROR_NONE && !s->seen_error) { + s->seen_error = true; + grpc_chttp2_maybe_complete_recv_trailing_metadata(exec_ctx, t, s); } - grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global, 1, - 1, due_to_error); + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, due_to_error); } -void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, - grpc_status_code status, gpr_slice *slice) { +void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_stream *s, grpc_status_code status, + grpc_slice *slice) { if (status != GRPC_STATUS_OK) { - stream_global->seen_error = true; - grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global, - stream_global); + s->seen_error = true; } /* stream_global->recv_trailing_metadata_finished gives us a last chance replacement: we've received trailing metadata, @@ -1574,27 +1469,26 @@ void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx, to the upper layers - drop what we've got, and then publish what we want - which is safe because we haven't told anyone about the metadata yet */ - if (!stream_global->published_trailing_metadata || - stream_global->recv_trailing_metadata_finished != NULL) { + if (s->published_metadata[1] == GRPC_METADATA_NOT_PUBLISHED || + s->recv_trailing_metadata_finished != NULL) { char status_string[GPR_LTOA_MIN_BUFSIZE]; gpr_ltoa(status, status_string); grpc_chttp2_incoming_metadata_buffer_add( - &stream_global->received_trailing_metadata, + &s->metadata_buffer[1], grpc_mdelem_from_metadata_strings( GRPC_MDSTR_GRPC_STATUS, grpc_mdstr_from_string(status_string))); if (slice) { grpc_chttp2_incoming_metadata_buffer_add( - &stream_global->received_trailing_metadata, + &s->metadata_buffer[1], grpc_mdelem_from_metadata_strings( GRPC_MDSTR_GRPC_MESSAGE, - grpc_mdstr_from_slice(gpr_slice_ref(*slice)))); + grpc_mdstr_from_slice(grpc_slice_ref(*slice)))); } - stream_global->published_trailing_metadata = true; - grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global, - stream_global); + s->published_metadata[1] = GRPC_METADATA_SYNTHESIZED_FROM_FAKE; + grpc_chttp2_maybe_complete_recv_trailing_metadata(exec_ctx, t, s); } if (slice) { - gpr_slice_unref(*slice); + grpc_slice_unref(*slice); } } @@ -1609,113 +1503,111 @@ static void add_error(grpc_error *error, grpc_error **refs, size_t *nrefs) { ++*nrefs; } -static grpc_error *removal_error(grpc_error *extra_error, - grpc_chttp2_stream_global *stream_global) { +static grpc_error *removal_error(grpc_error *extra_error, grpc_chttp2_stream *s, + const char *master_error_msg) { grpc_error *refs[3]; size_t nrefs = 0; - add_error(stream_global->read_closed_error, refs, &nrefs); - add_error(stream_global->write_closed_error, refs, &nrefs); + add_error(s->read_closed_error, refs, &nrefs); + add_error(s->write_closed_error, refs, &nrefs); add_error(extra_error, refs, &nrefs); grpc_error *error = GRPC_ERROR_NONE; if (nrefs > 0) { - error = GRPC_ERROR_CREATE_REFERENCING("Failed due to stream removal", refs, - nrefs); + error = GRPC_ERROR_CREATE_REFERENCING(master_error_msg, refs, nrefs); } GRPC_ERROR_UNREF(extra_error); return error; } static void fail_pending_writes(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, + grpc_chttp2_transport *t, grpc_chttp2_stream *s, grpc_error *error) { - error = removal_error(error, stream_global); - stream_global->send_message = NULL; + error = + removal_error(error, s, "Pending writes failed due to stream closure"); + s->fetching_send_message = NULL; + grpc_chttp2_complete_closure_step( + exec_ctx, t, s, &s->send_initial_metadata_finished, GRPC_ERROR_REF(error), + "send_initial_metadata_finished"); grpc_chttp2_complete_closure_step( - exec_ctx, transport_global, stream_global, - &stream_global->send_initial_metadata_finished, GRPC_ERROR_REF(error)); + exec_ctx, t, s, &s->send_trailing_metadata_finished, + GRPC_ERROR_REF(error), "send_trailing_metadata_finished"); grpc_chttp2_complete_closure_step( - exec_ctx, transport_global, stream_global, - &stream_global->send_trailing_metadata_finished, GRPC_ERROR_REF(error)); - grpc_chttp2_complete_closure_step(exec_ctx, transport_global, stream_global, - &stream_global->send_message_finished, - error); + exec_ctx, t, s, &s->fetching_send_message_finished, GRPC_ERROR_REF(error), + "fetching_send_message_finished"); + while (s->on_write_finished_cbs) { + grpc_chttp2_write_cb *cb = s->on_write_finished_cbs; + s->on_write_finished_cbs = cb->next; + grpc_chttp2_complete_closure_step(exec_ctx, t, s, &cb->closure, + GRPC_ERROR_REF(error), + "on_write_finished_cb"); + cb->next = t->write_cb_pool; + t->write_cb_pool = cb; + } + GRPC_ERROR_UNREF(error); } -void grpc_chttp2_mark_stream_closed( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, int close_reads, int close_writes, - grpc_error *error) { - if (stream_global->read_closed && stream_global->write_closed) { +void grpc_chttp2_mark_stream_closed(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, int close_reads, + int close_writes, grpc_error *error) { + if (s->read_closed && s->write_closed) { /* already closed */ GRPC_ERROR_UNREF(error); return; } - grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global, - stream_global); - if (close_reads && !stream_global->read_closed) { - stream_global->read_closed_error = GRPC_ERROR_REF(error); - stream_global->read_closed = true; - stream_global->published_initial_metadata = true; - stream_global->published_trailing_metadata = true; - decrement_active_streams_locked(exec_ctx, transport_global, stream_global); - } - if (close_writes && !stream_global->write_closed) { - stream_global->write_closed_error = GRPC_ERROR_REF(error); - stream_global->write_closed = true; - if (TRANSPORT_FROM_GLOBAL(transport_global)->executor.write_state != - GRPC_CHTTP2_WRITING_INACTIVE) { - GRPC_CHTTP2_STREAM_REF(stream_global, "finish_writes"); - grpc_chttp2_list_add_closed_waiting_for_writing(transport_global, - stream_global); - } else { - fail_pending_writes(exec_ctx, transport_global, stream_global, - GRPC_ERROR_REF(error)); - } - } - if (stream_global->read_closed && stream_global->write_closed) { - if (stream_global->id != 0 && - TRANSPORT_FROM_GLOBAL(transport_global)->executor.parsing_active) { - grpc_chttp2_list_add_closed_waiting_for_parsing(transport_global, - stream_global); - } else { - if (stream_global->id != 0) { - remove_stream(exec_ctx, TRANSPORT_FROM_GLOBAL(transport_global), - stream_global->id, - removal_error(GRPC_ERROR_REF(error), stream_global)); + if (close_reads && !s->read_closed) { + s->read_closed_error = GRPC_ERROR_REF(error); + s->read_closed = true; + for (int i = 0; i < 2; i++) { + if (s->published_metadata[i] == GRPC_METADATA_NOT_PUBLISHED) { + s->published_metadata[i] = GPRC_METADATA_PUBLISHED_AT_CLOSE; } - GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2"); } + decrement_active_streams_locked(exec_ctx, t, s); + grpc_chttp2_maybe_complete_recv_initial_metadata(exec_ctx, t, s); + grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s); + grpc_chttp2_maybe_complete_recv_trailing_metadata(exec_ctx, t, s); + } + if (close_writes && !s->write_closed) { + s->write_closed_error = GRPC_ERROR_REF(error); + s->write_closed = true; + fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error)); + grpc_chttp2_maybe_complete_recv_trailing_metadata(exec_ctx, t, s); + } + if (s->read_closed && s->write_closed) { + if (s->id != 0) { + remove_stream(exec_ctx, t, s->id, + removal_error(GRPC_ERROR_REF(error), s, "Stream removed")); + } + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2"); } GRPC_ERROR_UNREF(error); } -static void close_from_api(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, - grpc_error *error) { - gpr_slice hdr; - gpr_slice status_hdr; - gpr_slice message_pfx; +static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_stream *s, grpc_error *error) { + grpc_slice hdr; + grpc_slice status_hdr; + grpc_slice message_pfx; uint8_t *p; uint32_t len = 0; grpc_status_code grpc_status; grpc_chttp2_error_code http_error; - status_codes_from_error(error, stream_global->deadline, &http_error, - &grpc_status); + status_codes_from_error(error, s->deadline, &http_error, &grpc_status); GPR_ASSERT(grpc_status >= 0 && (int)grpc_status < 100); - if (stream_global->id != 0 && !transport_global->is_client) { + if (s->id != 0 && !t->is_client) { /* Hand roll a header block. - This is unnecessarily ugly - at some point we should find a more elegant + This is unnecessarily ugly - at some point we should find a more + elegant solution. - It's complicated by the fact that our send machinery would be dead by the + It's complicated by the fact that our send machinery would be dead by + the time we got around to sending this, so instead we ignore HPACK compression and just write the uncompressed bytes onto the wire. */ - status_hdr = gpr_slice_malloc(15 + (grpc_status >= 10)); - p = GPR_SLICE_START_PTR(status_hdr); + status_hdr = grpc_slice_malloc(15 + (grpc_status >= 10)); + p = GRPC_SLICE_START_PTR(status_hdr); *p++ = 0x40; /* literal header */ *p++ = 11; /* len(grpc-status) */ *p++ = 'g'; @@ -1737,8 +1629,8 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, *p++ = (uint8_t)('0' + (grpc_status / 10)); *p++ = (uint8_t)('0' + (grpc_status % 10)); } - GPR_ASSERT(p == GPR_SLICE_END_PTR(status_hdr)); - len += (uint32_t)GPR_SLICE_LENGTH(status_hdr); + GPR_ASSERT(p == GRPC_SLICE_END_PTR(status_hdr)); + len += (uint32_t)GRPC_SLICE_LENGTH(status_hdr); const char *optional_message = grpc_error_get_str(error, GRPC_ERROR_STR_GRPC_MESSAGE); @@ -1746,8 +1638,8 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, if (optional_message != NULL) { size_t msg_len = strlen(optional_message); GPR_ASSERT(msg_len < 127); - message_pfx = gpr_slice_malloc(15); - p = GPR_SLICE_START_PTR(message_pfx); + message_pfx = grpc_slice_malloc(15); + p = GRPC_SLICE_START_PTR(message_pfx); *p++ = 0x40; *p++ = 12; /* len(grpc-message) */ *p++ = 'g'; @@ -1763,35 +1655,34 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, *p++ = 'g'; *p++ = 'e'; *p++ = (uint8_t)msg_len; - GPR_ASSERT(p == GPR_SLICE_END_PTR(message_pfx)); - len += (uint32_t)GPR_SLICE_LENGTH(message_pfx); + GPR_ASSERT(p == GRPC_SLICE_END_PTR(message_pfx)); + len += (uint32_t)GRPC_SLICE_LENGTH(message_pfx); len += (uint32_t)msg_len; } - hdr = gpr_slice_malloc(9); - p = GPR_SLICE_START_PTR(hdr); + hdr = grpc_slice_malloc(9); + p = GRPC_SLICE_START_PTR(hdr); *p++ = (uint8_t)(len >> 16); *p++ = (uint8_t)(len >> 8); *p++ = (uint8_t)(len); *p++ = GRPC_CHTTP2_FRAME_HEADER; *p++ = GRPC_CHTTP2_DATA_FLAG_END_STREAM | GRPC_CHTTP2_DATA_FLAG_END_HEADERS; - *p++ = (uint8_t)(stream_global->id >> 24); - *p++ = (uint8_t)(stream_global->id >> 16); - *p++ = (uint8_t)(stream_global->id >> 8); - *p++ = (uint8_t)(stream_global->id); - GPR_ASSERT(p == GPR_SLICE_END_PTR(hdr)); - - gpr_slice_buffer_add(&transport_global->qbuf, hdr); - gpr_slice_buffer_add(&transport_global->qbuf, status_hdr); + *p++ = (uint8_t)(s->id >> 24); + *p++ = (uint8_t)(s->id >> 16); + *p++ = (uint8_t)(s->id >> 8); + *p++ = (uint8_t)(s->id); + GPR_ASSERT(p == GRPC_SLICE_END_PTR(hdr)); + + grpc_slice_buffer_add(&t->qbuf, hdr); + grpc_slice_buffer_add(&t->qbuf, status_hdr); if (optional_message) { - gpr_slice_buffer_add(&transport_global->qbuf, message_pfx); - gpr_slice_buffer_add(&transport_global->qbuf, - gpr_slice_from_copied_string(optional_message)); + grpc_slice_buffer_add(&t->qbuf, message_pfx); + grpc_slice_buffer_add(&t->qbuf, + grpc_slice_from_copied_string(optional_message)); } - gpr_slice_buffer_add( - &transport_global->qbuf, - grpc_chttp2_rst_stream_create(stream_global->id, GRPC_CHTTP2_NO_ERROR, - &stream_global->stats.outgoing)); + grpc_slice_buffer_add( + &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_CHTTP2_NO_ERROR, + &s->stats.outgoing)); } const char *msg = grpc_error_get_str(error, GRPC_ERROR_STR_GRPC_MESSAGE); @@ -1800,47 +1691,34 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, free_msg = true; msg = grpc_error_string(error); } - gpr_slice msg_slice = gpr_slice_from_copied_string(msg); - grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global, - grpc_status, &msg_slice); + grpc_slice msg_slice = grpc_slice_from_copied_string(msg); + grpc_chttp2_fake_status(exec_ctx, t, s, grpc_status, &msg_slice); if (free_msg) grpc_error_free_string(msg); - grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global, 1, - 1, error); - grpc_chttp2_initiate_write(exec_ctx, transport_global, false, - "close_from_api"); + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, 1, 1, error); + grpc_chttp2_initiate_write(exec_ctx, t, false, "close_from_api"); } typedef struct { grpc_exec_ctx *exec_ctx; grpc_error *error; + grpc_chttp2_transport *t; } cancel_stream_cb_args; -static void cancel_stream_cb(grpc_chttp2_transport_global *transport_global, - void *user_data, - grpc_chttp2_stream_global *stream_global) { +static void cancel_stream_cb(void *user_data, uint32_t key, void *stream) { cancel_stream_cb_args *args = user_data; - cancel_from_api(args->exec_ctx, transport_global, stream_global, - GRPC_ERROR_REF(args->error)); + grpc_chttp2_stream *s = stream; + grpc_chttp2_cancel_stream(args->exec_ctx, args->t, s, + GRPC_ERROR_REF(args->error)); } static void end_all_the_calls(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_error *error) { - cancel_stream_cb_args args = {exec_ctx, error}; - grpc_chttp2_for_all_streams(&t->global, &args, cancel_stream_cb); + cancel_stream_cb_args args = {exec_ctx, error, t}; + grpc_chttp2_stream_map_for_each(&t->stream_map, cancel_stream_cb, &args); GRPC_ERROR_UNREF(error); } -static void drop_connection(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, - grpc_error *error) { - if (!grpc_error_get_int(error, GRPC_ERROR_INT_GRPC_STATUS, NULL)) { - error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, - GRPC_STATUS_UNAVAILABLE); - } - close_transport_locked(exec_ctx, t, GRPC_ERROR_REF(error)); - end_all_the_calls(exec_ctx, t, error); -} - /** update window from a settings change */ typedef struct { grpc_chttp2_transport *t; @@ -1851,20 +1729,23 @@ static void update_global_window(void *args, uint32_t id, void *stream) { update_global_window_args *a = args; grpc_chttp2_transport *t = a->t; grpc_chttp2_stream *s = stream; - grpc_chttp2_transport_global *transport_global = &t->global; - grpc_chttp2_stream_global *stream_global = &s->global; int was_zero; int is_zero; - int64_t initial_window_update = t->parsing.initial_window_update; + int64_t initial_window_update = t->initial_window_update; - was_zero = stream_global->outgoing_window <= 0; - GRPC_CHTTP2_FLOW_CREDIT_STREAM("settings", transport_global, stream_global, - outgoing_window, initial_window_update); - is_zero = stream_global->outgoing_window <= 0; + if (initial_window_update > 0) { + was_zero = s->outgoing_window <= 0; + GRPC_CHTTP2_FLOW_CREDIT_STREAM("settings", t, s, outgoing_window, + initial_window_update); + is_zero = s->outgoing_window <= 0; - if (was_zero && !is_zero) { - grpc_chttp2_become_writable(a->exec_ctx, transport_global, stream_global, - true, "update_global_window"); + if (was_zero && !is_zero) { + grpc_chttp2_become_writable(a->exec_ctx, t, s, true, + "update_global_window"); + } + } else { + GRPC_CHTTP2_FLOW_DEBIT_STREAM("settings", t, s, outgoing_window, + -initial_window_update); } } @@ -1872,50 +1753,19 @@ static void update_global_window(void *args, uint32_t id, void *stream) { * INPUT PROCESSING - PARSING */ -static void parsing_action(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error); -static void post_reading_action_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error); -static void post_parse_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error); - -static void reading_action(grpc_exec_ctx *exec_ctx, void *tp, - grpc_error *error) { +static void read_action_begin(grpc_exec_ctx *exec_ctx, void *tp, + grpc_error *error) { /* Control flow: reading_action_locked -> (parse_unlocked -> post_parse_locked)? -> post_reading_action_locked */ GPR_TIMER_BEGIN("reading_action", 0); grpc_chttp2_transport *t = tp; - grpc_combiner_execute(exec_ctx, t->executor.combiner, - &t->reading_action_locked, GRPC_ERROR_REF(error)); + grpc_combiner_execute(exec_ctx, t->combiner, &t->read_action_locked, + GRPC_ERROR_REF(error), false); GPR_TIMER_END("reading_action", 0); } -static void reading_action_locked(grpc_exec_ctx *exec_ctx, void *tp, - grpc_error *error) { - GPR_TIMER_BEGIN("reading_action_locked", 0); - - grpc_chttp2_transport *t = tp; - grpc_chttp2_transport_global *transport_global = &t->global; - grpc_chttp2_transport_parsing *transport_parsing = &t->parsing; - - GPR_ASSERT(!t->executor.parsing_active); - if (!t->closed) { - t->executor.parsing_active = 1; - /* merge stream lists */ - grpc_chttp2_stream_map_move_into(&t->new_stream_map, - &t->parsing_stream_map); - grpc_chttp2_prepare_to_read(transport_global, transport_parsing); - grpc_exec_ctx_sched(exec_ctx, &t->parsing_action, GRPC_ERROR_REF(error), - NULL); - } else { - post_reading_action_locked(exec_ctx, t, error); - } - - GPR_TIMER_END("reading_action_locked", 0); -} - static grpc_error *try_http_parsing(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { grpc_http_parser parser; @@ -1944,131 +1794,102 @@ static grpc_error *try_http_parsing(grpc_exec_ctx *exec_ctx, return error; } -static void parsing_action(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { - grpc_chttp2_transport *t = arg; - grpc_error *err = GRPC_ERROR_NONE; - GPR_TIMER_BEGIN("reading_action.parse", 0); - size_t i = 0; - grpc_error *errors[3] = {GRPC_ERROR_REF(error), GRPC_ERROR_NONE, - GRPC_ERROR_NONE}; - for (; i < t->read_buffer.count && errors[1] == GRPC_ERROR_NONE; i++) { - errors[1] = grpc_chttp2_perform_read(exec_ctx, &t->parsing, - t->read_buffer.slices[i]); - }; - if (errors[1] == GRPC_ERROR_NONE) { - err = GRPC_ERROR_REF(error); - } else { - errors[2] = try_http_parsing(exec_ctx, t); - err = GRPC_ERROR_CREATE_REFERENCING("Failed parsing HTTP/2", errors, - GPR_ARRAY_SIZE(errors)); +static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp, + grpc_error *error) { + GPR_TIMER_BEGIN("reading_action_locked", 0); + + grpc_chttp2_transport *t = tp; + + GRPC_ERROR_REF(error); + + grpc_error *err = error; + if (err != GRPC_ERROR_NONE) { + err = grpc_error_set_int( + GRPC_ERROR_CREATE_REFERENCING("Endpoint read failed", &err, 1), + GRPC_ERROR_INT_OCCURRED_DURING_WRITE, t->write_state); } - for (i = 0; i < GPR_ARRAY_SIZE(errors); i++) { - GRPC_ERROR_UNREF(errors[i]); + GPR_SWAP(grpc_error *, err, error); + GRPC_ERROR_UNREF(err); + if (!t->closed) { + GPR_TIMER_BEGIN("reading_action.parse", 0); + size_t i = 0; + grpc_error *errors[3] = {GRPC_ERROR_REF(error), GRPC_ERROR_NONE, + GRPC_ERROR_NONE}; + for (; i < t->read_buffer.count && errors[1] == GRPC_ERROR_NONE; i++) { + errors[1] = + grpc_chttp2_perform_read(exec_ctx, t, t->read_buffer.slices[i]); + }; + if (errors[1] != GRPC_ERROR_NONE) { + errors[2] = try_http_parsing(exec_ctx, t); + GRPC_ERROR_UNREF(error); + error = GRPC_ERROR_CREATE_REFERENCING("Failed parsing HTTP/2", errors, + GPR_ARRAY_SIZE(errors)); + } + for (i = 0; i < GPR_ARRAY_SIZE(errors); i++) { + GRPC_ERROR_UNREF(errors[i]); + } + GPR_TIMER_END("reading_action.parse", 0); + + GPR_TIMER_BEGIN("post_parse_locked", 0); + if (t->initial_window_update != 0) { + update_global_window_args args = {t, exec_ctx}; + grpc_chttp2_stream_map_for_each(&t->stream_map, update_global_window, + &args); + t->initial_window_update = 0; + } + /* handle higher level things */ + if (t->incoming_window < t->connection_window_target * 3 / 4) { + int64_t announce_bytes = t->connection_window_target - t->incoming_window; + GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parsed", t, announce_incoming_window, + announce_bytes); + GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parsed", t, incoming_window, + announce_bytes); + grpc_chttp2_initiate_write(exec_ctx, t, false, "global incoming window"); + } + + GPR_TIMER_END("post_parse_locked", 0); } - grpc_combiner_execute(exec_ctx, t->executor.combiner, &t->post_parse_locked, - err); - GPR_TIMER_END("reading_action.parse", 0); -} -static void post_parse_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { - GPR_TIMER_BEGIN("post_parse_locked", 0); - grpc_chttp2_transport *t = arg; - grpc_chttp2_transport_global *transport_global = &t->global; - grpc_chttp2_transport_parsing *transport_parsing = &t->parsing; - /* copy parsing qbuf to global qbuf */ - if (t->parsing.qbuf.count > 0) { - gpr_slice_buffer_move_into(&t->parsing.qbuf, &t->global.qbuf); - grpc_chttp2_initiate_write(exec_ctx, transport_global, false, - "parsing_qbuf"); - } - /* merge stream lists */ - grpc_chttp2_stream_map_move_into(&t->new_stream_map, &t->parsing_stream_map); - transport_global->concurrent_stream_count = - (uint32_t)grpc_chttp2_stream_map_size(&t->parsing_stream_map); - if (transport_parsing->initial_window_update != 0) { - update_global_window_args args = {t, exec_ctx}; - grpc_chttp2_stream_map_for_each(&t->parsing_stream_map, - update_global_window, &args); - transport_parsing->initial_window_update = 0; - } - /* handle higher level things */ - grpc_chttp2_publish_reads(exec_ctx, transport_global, transport_parsing); - t->executor.parsing_active = 0; - /* handle delayed transport ops (if there is one) */ - if (t->post_parsing_op) { - grpc_transport_op *op = t->post_parsing_op; - t->post_parsing_op = NULL; - perform_transport_op_locked(exec_ctx, op, GRPC_ERROR_NONE); - gpr_free(op); - } - /* if a stream is in the stream map, and gets cancelled, we need to - * ensure we are not parsing before continuing the cancellation to keep - * things in a sane state */ - grpc_chttp2_stream_global *stream_global; - while (grpc_chttp2_list_pop_closed_waiting_for_parsing(transport_global, - &stream_global)) { - GPR_ASSERT(stream_global->in_stream_map); - GPR_ASSERT(stream_global->write_closed); - GPR_ASSERT(stream_global->read_closed); - remove_stream(exec_ctx, t, stream_global->id, - removal_error(GRPC_ERROR_NONE, stream_global)); - GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2"); - } - - post_reading_action_locked(exec_ctx, t, error); - GPR_TIMER_END("post_parse_locked", 0); -} - -static void post_reading_action_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { GPR_TIMER_BEGIN("post_reading_action_locked", 0); - grpc_chttp2_transport *t = arg; bool keep_reading = false; - GRPC_ERROR_REF(error); if (error == GRPC_ERROR_NONE && t->closed) { error = GRPC_ERROR_CREATE("Transport closed"); } if (error != GRPC_ERROR_NONE) { - drop_connection(exec_ctx, t, GRPC_ERROR_REF(error)); + close_transport_locked(exec_ctx, t, GRPC_ERROR_REF(error)); t->endpoint_reading = 0; - if (grpc_http_write_state_trace) { - gpr_log(GPR_DEBUG, "R:%p -> 0 ws=%s", t, - write_state_name(t->executor.write_state)); - } } else if (!t->closed) { keep_reading = true; - REF_TRANSPORT(t, "keep_reading"); - prevent_endpoint_shutdown(t); + GRPC_CHTTP2_REF_TRANSPORT(t, "keep_reading"); } - gpr_slice_buffer_reset_and_unref(&t->read_buffer); + grpc_slice_buffer_reset_and_unref(&t->read_buffer); if (keep_reading) { - grpc_endpoint_read(exec_ctx, t->ep, &t->read_buffer, &t->reading_action); - allow_endpoint_shutdown_locked(exec_ctx, t); - UNREF_TRANSPORT(exec_ctx, t, "keep_reading"); + grpc_endpoint_read(exec_ctx, t->ep, &t->read_buffer, &t->read_action_begin); + GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "keep_reading"); } else { - UNREF_TRANSPORT(exec_ctx, t, "reading_action"); + GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "reading_action"); } - GRPC_ERROR_UNREF(error); GPR_TIMER_END("post_reading_action_locked", 0); + + GRPC_ERROR_UNREF(error); + + GPR_TIMER_END("reading_action_locked", 0); } /******************************************************************************* * CALLBACK LOOP */ -static void connectivity_state_set( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_connectivity_state state, grpc_error *error, const char *reason) { +static void connectivity_state_set(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_connectivity_state state, + grpc_error *error, const char *reason) { GRPC_CHTTP2_IF_TRACING( gpr_log(GPR_DEBUG, "set connectivity_state=%d", state)); - grpc_connectivity_state_set( - exec_ctx, - &TRANSPORT_FROM_GLOBAL(transport_global)->channel_callback.state_tracker, - state, error, reason); + grpc_connectivity_state_set(exec_ctx, &t->channel_callback.state_tracker, + state, error, reason); } /******************************************************************************* @@ -2095,21 +1916,22 @@ static void incoming_byte_stream_unref(grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs) { if (gpr_unref(&bs->refs)) { GRPC_ERROR_UNREF(bs->error); - gpr_slice_buffer_destroy(&bs->slices); + grpc_slice_buffer_destroy(&bs->slices); gpr_mu_destroy(&bs->slice_mu); gpr_free(bs); } } -static void incoming_byte_stream_update_flow_control( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, size_t max_size_hint, - size_t have_already) { +static void incoming_byte_stream_update_flow_control(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + size_t max_size_hint, + size_t have_already) { uint32_t max_recv_bytes; /* clamp max recv hint to an allowable size */ - if (max_size_hint >= UINT32_MAX - transport_global->stream_lookahead) { - max_recv_bytes = UINT32_MAX - transport_global->stream_lookahead; + if (max_size_hint >= UINT32_MAX - t->stream_lookahead) { + max_recv_bytes = UINT32_MAX - t->stream_lookahead; } else { max_recv_bytes = (uint32_t)max_size_hint; } @@ -2122,23 +1944,21 @@ static void incoming_byte_stream_update_flow_control( } /* add some small lookahead to keep pipelines flowing */ - GPR_ASSERT(max_recv_bytes <= UINT32_MAX - transport_global->stream_lookahead); - max_recv_bytes += transport_global->stream_lookahead; - if (stream_global->max_recv_bytes < max_recv_bytes) { - uint32_t add_max_recv_bytes = - max_recv_bytes - stream_global->max_recv_bytes; - GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", transport_global, stream_global, - max_recv_bytes, add_max_recv_bytes); - GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", transport_global, stream_global, - unannounced_incoming_window_for_parse, + GPR_ASSERT(max_recv_bytes <= UINT32_MAX - t->stream_lookahead); + max_recv_bytes += t->stream_lookahead; + if (s->max_recv_bytes < max_recv_bytes) { + uint32_t add_max_recv_bytes = max_recv_bytes - s->max_recv_bytes; + bool new_window_write_is_covered_by_poller = + s->max_recv_bytes < have_already; + GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", t, s, max_recv_bytes, + add_max_recv_bytes); + GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", t, s, incoming_window, add_max_recv_bytes); - GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", transport_global, stream_global, - unannounced_incoming_window_for_writing, + GRPC_CHTTP2_FLOW_CREDIT_STREAM("op", t, s, announce_window, add_max_recv_bytes); - grpc_chttp2_list_add_unannounced_incoming_window_available(transport_global, - stream_global); - grpc_chttp2_become_writable(exec_ctx, transport_global, stream_global, - false, "read_incoming_stream"); + grpc_chttp2_become_writable(exec_ctx, t, s, + new_window_write_is_covered_by_poller, + "read_incoming_stream"); } } @@ -2146,25 +1966,23 @@ static void incoming_byte_stream_next_locked(grpc_exec_ctx *exec_ctx, void *argp, grpc_error *error_ignored) { grpc_chttp2_incoming_byte_stream *bs = argp; - grpc_chttp2_transport_global *transport_global = &bs->transport->global; - grpc_chttp2_stream_global *stream_global = &bs->stream->global; + grpc_chttp2_transport *t = bs->transport; + grpc_chttp2_stream *s = bs->stream; if (bs->is_tail) { gpr_mu_lock(&bs->slice_mu); size_t cur_length = bs->slices.length; gpr_mu_unlock(&bs->slice_mu); incoming_byte_stream_update_flow_control( - exec_ctx, transport_global, stream_global, - bs->next_action.max_size_hint, cur_length); + exec_ctx, t, s, bs->next_action.max_size_hint, cur_length); } gpr_mu_lock(&bs->slice_mu); if (bs->slices.count > 0) { - *bs->next_action.slice = gpr_slice_buffer_take_first(&bs->slices); - grpc_exec_ctx_sched(exec_ctx, bs->next_action.on_complete, GRPC_ERROR_NONE, - NULL); + *bs->next_action.slice = grpc_slice_buffer_take_first(&bs->slices); + grpc_closure_run(exec_ctx, bs->next_action.on_complete, GRPC_ERROR_NONE); } else if (bs->error != GRPC_ERROR_NONE) { - grpc_exec_ctx_sched(exec_ctx, bs->next_action.on_complete, - GRPC_ERROR_REF(bs->error), NULL); + grpc_closure_run(exec_ctx, bs->next_action.on_complete, + GRPC_ERROR_REF(bs->error)); } else { bs->on_next = bs->next_action.on_complete; bs->next = bs->next_action.slice; @@ -2175,7 +1993,7 @@ static void incoming_byte_stream_next_locked(grpc_exec_ctx *exec_ctx, static int incoming_byte_stream_next(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream, - gpr_slice *slice, size_t max_size_hint, + grpc_slice *slice, size_t max_size_hint, grpc_closure *on_complete) { GPR_TIMER_BEGIN("incoming_byte_stream_next", 0); grpc_chttp2_incoming_byte_stream *bs = @@ -2186,8 +2004,8 @@ static int incoming_byte_stream_next(grpc_exec_ctx *exec_ctx, bs->next_action.on_complete = on_complete; grpc_closure_init(&bs->next_action.closure, incoming_byte_stream_next_locked, bs); - grpc_combiner_execute(exec_ctx, bs->transport->executor.combiner, - &bs->next_action.closure, GRPC_ERROR_NONE); + grpc_combiner_execute(exec_ctx, bs->transport->combiner, + &bs->next_action.closure, GRPC_ERROR_NONE, false); GPR_TIMER_END("incoming_byte_stream_next", 0); return 0; } @@ -2200,8 +2018,7 @@ static void incoming_byte_stream_destroy_locked(grpc_exec_ctx *exec_ctx, grpc_error *error_ignored) { grpc_chttp2_incoming_byte_stream *bs = byte_stream; GPR_ASSERT(bs->base.destroy == incoming_byte_stream_destroy); - decrement_active_streams_locked(exec_ctx, &bs->transport->global, - &bs->stream->global); + decrement_active_streams_locked(exec_ctx, bs->transport, bs->stream); incoming_byte_stream_unref(exec_ctx, bs); } @@ -2212,126 +2029,215 @@ static void incoming_byte_stream_destroy(grpc_exec_ctx *exec_ctx, (grpc_chttp2_incoming_byte_stream *)byte_stream; grpc_closure_init(&bs->destroy_action, incoming_byte_stream_destroy_locked, bs); - grpc_combiner_execute(exec_ctx, bs->transport->executor.combiner, - &bs->destroy_action, GRPC_ERROR_NONE); + grpc_combiner_execute(exec_ctx, bs->transport->combiner, &bs->destroy_action, + GRPC_ERROR_NONE, false); GPR_TIMER_END("incoming_byte_stream_destroy", 0); } -typedef struct { - grpc_chttp2_incoming_byte_stream *byte_stream; - gpr_slice slice; -} incoming_byte_stream_push_arg; +static void incoming_byte_stream_publish_error( + grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, + grpc_error *error) { + GPR_ASSERT(error != GRPC_ERROR_NONE); + grpc_exec_ctx_sched(exec_ctx, bs->on_next, GRPC_ERROR_REF(error), NULL); + bs->on_next = NULL; + GRPC_ERROR_UNREF(bs->error); + bs->error = error; +} void grpc_chttp2_incoming_byte_stream_push(grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, - gpr_slice slice) { + grpc_slice slice) { gpr_mu_lock(&bs->slice_mu); - if (bs->on_next != NULL) { - *bs->next = slice; - grpc_exec_ctx_sched(exec_ctx, bs->on_next, GRPC_ERROR_NONE, NULL); - bs->on_next = NULL; + if (bs->remaining_bytes < GRPC_SLICE_LENGTH(slice)) { + incoming_byte_stream_publish_error( + exec_ctx, bs, GRPC_ERROR_CREATE("Too many bytes in stream")); } else { - gpr_slice_buffer_add(&bs->slices, slice); + bs->remaining_bytes -= (uint32_t)GRPC_SLICE_LENGTH(slice); + if (bs->on_next != NULL) { + *bs->next = slice; + grpc_exec_ctx_sched(exec_ctx, bs->on_next, GRPC_ERROR_NONE, NULL); + bs->on_next = NULL; + } else { + grpc_slice_buffer_add(&bs->slices, slice); + } } gpr_mu_unlock(&bs->slice_mu); } -static void incoming_byte_stream_finished_locked(grpc_exec_ctx *exec_ctx, - void *bsp, grpc_error *error) { - grpc_chttp2_incoming_byte_stream *bs = bsp; - if (error != GRPC_ERROR_NONE) { - grpc_exec_ctx_sched(exec_ctx, bs->on_next, GRPC_ERROR_REF(error), NULL); - bs->on_next = NULL; - GRPC_ERROR_UNREF(bs->error); - bs->error = error; - } - incoming_byte_stream_unref(exec_ctx, bs); -} - void grpc_chttp2_incoming_byte_stream_finished( grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, - grpc_error *error, int from_parsing_thread) { - GPR_TIMER_BEGIN("grpc_chttp2_incoming_byte_stream_finished", 0); - if (from_parsing_thread) { - grpc_closure_init(&bs->finished_action, - incoming_byte_stream_finished_locked, bs); - grpc_combiner_execute(exec_ctx, bs->transport->executor.combiner, - &bs->finished_action, GRPC_ERROR_REF(error)); - } else { - incoming_byte_stream_finished_locked(exec_ctx, bs, error); + grpc_error *error) { + if (error == GRPC_ERROR_NONE) { + gpr_mu_lock(&bs->slice_mu); + if (bs->remaining_bytes != 0) { + error = GRPC_ERROR_CREATE("Truncated message"); + } + gpr_mu_unlock(&bs->slice_mu); + } + if (error != GRPC_ERROR_NONE) { + incoming_byte_stream_publish_error(exec_ctx, bs, error); } - GPR_TIMER_END("grpc_chttp2_incoming_byte_stream_finished", 0); + incoming_byte_stream_unref(exec_ctx, bs); } grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, uint32_t frame_size, - uint32_t flags, grpc_chttp2_incoming_frame_queue *add_to_queue) { + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, + uint32_t frame_size, uint32_t flags) { grpc_chttp2_incoming_byte_stream *incoming_byte_stream = gpr_malloc(sizeof(*incoming_byte_stream)); incoming_byte_stream->base.length = frame_size; + incoming_byte_stream->remaining_bytes = frame_size; incoming_byte_stream->base.flags = flags; incoming_byte_stream->base.next = incoming_byte_stream_next; incoming_byte_stream->base.destroy = incoming_byte_stream_destroy; gpr_mu_init(&incoming_byte_stream->slice_mu); gpr_ref_init(&incoming_byte_stream->refs, 2); incoming_byte_stream->next_message = NULL; - incoming_byte_stream->transport = TRANSPORT_FROM_PARSING(transport_parsing); - incoming_byte_stream->stream = STREAM_FROM_PARSING(stream_parsing); - gpr_ref(&incoming_byte_stream->stream->global.active_streams); - gpr_slice_buffer_init(&incoming_byte_stream->slices); + incoming_byte_stream->transport = t; + incoming_byte_stream->stream = s; + gpr_ref(&incoming_byte_stream->stream->active_streams); + grpc_slice_buffer_init(&incoming_byte_stream->slices); incoming_byte_stream->on_next = NULL; incoming_byte_stream->is_tail = 1; incoming_byte_stream->error = GRPC_ERROR_NONE; - if (add_to_queue->head == NULL) { - add_to_queue->head = incoming_byte_stream; + grpc_chttp2_incoming_frame_queue *q = &s->incoming_frames; + if (q->head == NULL) { + q->head = incoming_byte_stream; } else { - add_to_queue->tail->is_tail = 0; - add_to_queue->tail->next_message = incoming_byte_stream; + q->tail->is_tail = 0; + q->tail->next_message = incoming_byte_stream; } - add_to_queue->tail = incoming_byte_stream; + q->tail = incoming_byte_stream; + grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s); return incoming_byte_stream; } /******************************************************************************* - * TRACING + * RESOURCE QUOTAS */ -static char *format_flowctl_context_var(const char *context, const char *var, - int64_t val, uint32_t id, - char **scope) { - char *underscore_pos; - char *buf; - char *result; - if (context == NULL) { - *scope = NULL; - gpr_asprintf(&buf, "%s(%" PRId64 ")", var, val); - result = gpr_leftpad(buf, ' ', 60); - gpr_free(buf); - return result; - } - underscore_pos = strchr(context, '_'); - *scope = gpr_strdup(context); - (*scope)[underscore_pos - context] = 0; - if (id != 0) { - char *tmp = *scope; - gpr_asprintf(scope, "%s[%d]", tmp, id); - gpr_free(tmp); - } - gpr_asprintf(&buf, "%s.%s(%" PRId64 ")", underscore_pos + 1, var, val); - result = gpr_leftpad(buf, ' ', 60); - gpr_free(buf); - return result; +static void post_benign_reclaimer(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { + if (!t->benign_reclaimer_registered) { + t->benign_reclaimer_registered = true; + GRPC_CHTTP2_REF_TRANSPORT(t, "benign_reclaimer"); + grpc_resource_user_post_reclaimer(exec_ctx, + grpc_endpoint_get_resource_user(t->ep), + false, &t->benign_reclaimer); + } +} + +static void post_destructive_reclaimer(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { + if (!t->destructive_reclaimer_registered) { + t->destructive_reclaimer_registered = true; + GRPC_CHTTP2_REF_TRANSPORT(t, "destructive_reclaimer"); + grpc_resource_user_post_reclaimer(exec_ctx, + grpc_endpoint_get_resource_user(t->ep), + true, &t->destructive_reclaimer); + } +} + +static void benign_reclaimer(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_chttp2_transport *t = arg; + grpc_combiner_execute(exec_ctx, t->combiner, &t->benign_reclaimer_locked, + GRPC_ERROR_REF(error), false); +} + +static void destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_chttp2_transport *t = arg; + grpc_combiner_execute(exec_ctx, t->combiner, &t->destructive_reclaimer_locked, + GRPC_ERROR_REF(error), false); +} + +static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_chttp2_transport *t = arg; + if (error == GRPC_ERROR_NONE && + grpc_chttp2_stream_map_size(&t->stream_map) == 0) { + /* Channel with no active streams: send a goaway to try and make it + * disconnect cleanly */ + if (grpc_resource_quota_trace) { + gpr_log(GPR_DEBUG, "HTTP2: %s - send goaway to free memory", + t->peer_string); + } + send_goaway(exec_ctx, t, GRPC_CHTTP2_ENHANCE_YOUR_CALM, + grpc_slice_from_static_string("Buffers full")); + } else if (error == GRPC_ERROR_NONE && grpc_resource_quota_trace) { + gpr_log(GPR_DEBUG, + "HTTP2: %s - skip benign reclamation, there are still %" PRIdPTR + " streams", + t->peer_string, grpc_chttp2_stream_map_size(&t->stream_map)); + } + t->benign_reclaimer_registered = false; + if (error != GRPC_ERROR_CANCELLED) { + grpc_resource_user_finish_reclamation( + exec_ctx, grpc_endpoint_get_resource_user(t->ep)); + } + GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "benign_reclaimer"); } -static int samestr(char *a, char *b) { - if (a == NULL) { - return b == NULL; +static void destructive_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_chttp2_transport *t = arg; + size_t n = grpc_chttp2_stream_map_size(&t->stream_map); + t->destructive_reclaimer_registered = false; + if (error == GRPC_ERROR_NONE && n > 0) { + grpc_chttp2_stream *s = grpc_chttp2_stream_map_rand(&t->stream_map); + if (grpc_resource_quota_trace) { + gpr_log(GPR_DEBUG, "HTTP2: %s - abandon stream id %d", t->peer_string, + s->id); + } + grpc_chttp2_cancel_stream( + exec_ctx, t, s, grpc_error_set_int(GRPC_ERROR_CREATE("Buffers full"), + GRPC_ERROR_INT_HTTP2_ERROR, + GRPC_CHTTP2_ENHANCE_YOUR_CALM)); + if (n > 1) { + /* Since we cancel one stream per destructive reclamation, if + there are more streams left, we can immediately post a new + reclaimer in case the resource quota needs to free more + memory */ + post_destructive_reclaimer(exec_ctx, t); + } } - if (b == NULL) { - return 0; + if (error != GRPC_ERROR_CANCELLED) { + grpc_resource_user_finish_reclamation( + exec_ctx, grpc_endpoint_get_resource_user(t->ep)); + } + GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "destructive_reclaimer"); +} + +/******************************************************************************* + * TRACING + */ + +static char *format_flowctl_context_var(const char *context, const char *var, + int64_t val, uint32_t id) { + char *name; + if (context == NULL) { + name = gpr_strdup(var); + } else if (0 == strcmp(context, "t")) { + GPR_ASSERT(id == 0); + gpr_asprintf(&name, "TRANSPORT:%s", var); + } else if (0 == strcmp(context, "s")) { + GPR_ASSERT(id != 0); + gpr_asprintf(&name, "STREAM[%d]:%s", id, var); + } else { + gpr_asprintf(&name, "BAD_CONTEXT[%s][%d]:%s", context, id, var); } - return 0 == strcmp(a, b); + char *name_fld = gpr_leftpad(name, ' ', 64); + char *value; + gpr_asprintf(&value, "%" PRId64, val); + char *value_fld = gpr_leftpad(value, ' ', 8); + char *result; + gpr_asprintf(&result, "%s %s", name_fld, value_fld); + gpr_free(name); + gpr_free(name_fld); + gpr_free(value); + gpr_free(value_fld); + return result; } void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase, @@ -2339,26 +2245,18 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase, const char *var1, const char *context2, const char *var2, int is_client, uint32_t stream_id, int64_t val1, int64_t val2) { - char *scope1; - char *scope2; char *tmp_phase; - char *tmp_scope1; - char *label1 = - format_flowctl_context_var(context1, var1, val1, stream_id, &scope1); - char *label2 = - format_flowctl_context_var(context2, var2, val2, stream_id, &scope2); + char *label1 = format_flowctl_context_var(context1, var1, val1, stream_id); + char *label2 = format_flowctl_context_var(context2, var2, val2, stream_id); char *clisvr = is_client ? "client" : "server"; char *prefix; tmp_phase = gpr_leftpad(phase, ' ', 8); - tmp_scope1 = gpr_leftpad(scope1, ' ', 11); - gpr_asprintf(&prefix, "FLOW %s: %s %s ", tmp_phase, clisvr, scope1); + gpr_asprintf(&prefix, "FLOW %s: %s ", tmp_phase, clisvr); gpr_free(tmp_phase); - gpr_free(tmp_scope1); switch (op) { case GRPC_CHTTP2_FLOWCTL_MOVE: - GPR_ASSERT(samestr(scope1, scope2)); if (val2 != 0) { gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "%sMOVE %s <- %s giving %" PRId64, prefix, label1, label2, @@ -2383,8 +2281,6 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase, break; } - gpr_free(scope1); - gpr_free(scope2); gpr_free(label1); gpr_free(label2); gpr_free(prefix); @@ -2419,12 +2315,13 @@ grpc_transport *grpc_create_chttp2_transport( void grpc_chttp2_transport_start_reading(grpc_exec_ctx *exec_ctx, grpc_transport *transport, - gpr_slice_buffer *read_buffer) { + grpc_slice_buffer *read_buffer) { grpc_chttp2_transport *t = (grpc_chttp2_transport *)transport; - REF_TRANSPORT(t, "reading_action"); /* matches unref inside reading_action */ + GRPC_CHTTP2_REF_TRANSPORT( + t, "reading_action"); /* matches unref inside reading_action */ if (read_buffer != NULL) { - gpr_slice_buffer_move_into(read_buffer, &t->read_buffer); + grpc_slice_buffer_move_into(read_buffer, &t->read_buffer); gpr_free(read_buffer); } - reading_action(exec_ctx, t, GRPC_ERROR_NONE); + read_action_begin(exec_ctx, t, GRPC_ERROR_NONE); } diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.h b/src/core/ext/transport/chttp2/transport/chttp2_transport.h index 4e2d0954bf..c372174f2d 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.h +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.h @@ -48,6 +48,6 @@ grpc_transport *grpc_create_chttp2_transport( /// leftover bytes previously read from the endpoint (e.g., by handshakers). void grpc_chttp2_transport_start_reading(grpc_exec_ctx *exec_ctx, grpc_transport *transport, - gpr_slice_buffer *read_buffer); + grpc_slice_buffer *read_buffer); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_CHTTP2_TRANSPORT_H */ diff --git a/src/core/ext/transport/chttp2/transport/frame.h b/src/core/ext/transport/chttp2/transport/frame.h index 507aae4100..ffd4d9669b 100644 --- a/src/core/ext/transport/chttp2/transport/frame.h +++ b/src/core/ext/transport/chttp2/transport/frame.h @@ -34,14 +34,14 @@ #ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_H #define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_H +#include <grpc/slice.h> #include <grpc/support/port_platform.h> -#include <grpc/support/slice.h> #include "src/core/lib/iomgr/error.h" /* defined in internal.h */ -typedef struct grpc_chttp2_stream_parsing grpc_chttp2_stream_parsing; -typedef struct grpc_chttp2_transport_parsing grpc_chttp2_transport_parsing; +typedef struct grpc_chttp2_stream grpc_chttp2_stream; +typedef struct grpc_chttp2_transport grpc_chttp2_transport; #define GRPC_CHTTP2_FRAME_DATA 0 #define GRPC_CHTTP2_FRAME_HEADER 1 diff --git a/src/core/ext/transport/chttp2/transport/frame_data.c b/src/core/ext/transport/chttp2/transport/frame_data.c index 9046fbc453..f9b9e1b309 100644 --- a/src/core/ext/transport/chttp2/transport/frame_data.c +++ b/src/core/ext/transport/chttp2/transport/frame_data.c @@ -40,6 +40,7 @@ #include <grpc/support/string_util.h> #include <grpc/support/useful.h> #include "src/core/ext/transport/chttp2/transport/internal.h" +#include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" #include "src/core/lib/transport/transport.h" @@ -51,16 +52,11 @@ grpc_error *grpc_chttp2_data_parser_init(grpc_chttp2_data_parser *parser) { void grpc_chttp2_data_parser_destroy(grpc_exec_ctx *exec_ctx, grpc_chttp2_data_parser *parser) { - grpc_byte_stream *bs; - if (parser->parsing_frame) { + if (parser->parsing_frame != NULL) { grpc_chttp2_incoming_byte_stream_finished( - exec_ctx, parser->parsing_frame, GRPC_ERROR_CREATE("Parser destroyed"), - 1); - } - while ( - (bs = grpc_chttp2_incoming_frame_queue_pop(&parser->incoming_frames))) { - grpc_byte_stream_destroy(exec_ctx, bs); + exec_ctx, parser->parsing_frame, GRPC_ERROR_CREATE("Parser destroyed")); } + GRPC_ERROR_UNREF(parser->error); } grpc_error *grpc_chttp2_data_parser_begin_frame(grpc_chttp2_data_parser *parser, @@ -117,16 +113,16 @@ grpc_byte_stream *grpc_chttp2_incoming_frame_queue_pop( return out; } -void grpc_chttp2_encode_data(uint32_t id, gpr_slice_buffer *inbuf, +void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer *inbuf, uint32_t write_bytes, int is_eof, grpc_transport_one_way_stats *stats, - gpr_slice_buffer *outbuf) { - gpr_slice hdr; + grpc_slice_buffer *outbuf) { + grpc_slice hdr; uint8_t *p; static const size_t header_size = 9; - hdr = gpr_slice_malloc(header_size); - p = GPR_SLICE_START_PTR(hdr); + hdr = grpc_slice_malloc(header_size); + p = GRPC_SLICE_START_PTR(hdr); GPR_ASSERT(write_bytes < (1 << 24)); *p++ = (uint8_t)(write_bytes >> 16); *p++ = (uint8_t)(write_bytes >> 8); @@ -137,30 +133,25 @@ void grpc_chttp2_encode_data(uint32_t id, gpr_slice_buffer *inbuf, *p++ = (uint8_t)(id >> 16); *p++ = (uint8_t)(id >> 8); *p++ = (uint8_t)(id); - gpr_slice_buffer_add(outbuf, hdr); + grpc_slice_buffer_add(outbuf, hdr); - gpr_slice_buffer_move_first(inbuf, write_bytes, outbuf); + grpc_slice_buffer_move_first(inbuf, write_bytes, outbuf); stats->framing_bytes += header_size; stats->data_bytes += write_bytes; } -grpc_error *grpc_chttp2_data_parser_parse( - grpc_exec_ctx *exec_ctx, void *parser, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) { - uint8_t *const beg = GPR_SLICE_START_PTR(slice); - uint8_t *const end = GPR_SLICE_END_PTR(slice); +static grpc_error *parse_inner(grpc_exec_ctx *exec_ctx, + grpc_chttp2_data_parser *p, + grpc_chttp2_transport *t, grpc_chttp2_stream *s, + grpc_slice slice) { + uint8_t *const beg = GRPC_SLICE_START_PTR(slice); + uint8_t *const end = GRPC_SLICE_END_PTR(slice); uint8_t *cur = beg; - grpc_chttp2_data_parser *p = parser; uint32_t message_flags; grpc_chttp2_incoming_byte_stream *incoming_byte_stream; char *msg; - if (is_last && p->is_last_frame) { - stream_parsing->received_close = 1; - } - if (cur == end) { return GRPC_ERROR_NONE; } @@ -171,7 +162,7 @@ grpc_error *grpc_chttp2_data_parser_parse( return GRPC_ERROR_REF(p->error); fh_0: case GRPC_CHTTP2_DATA_FH_0: - stream_parsing->stats.incoming.framing_bytes++; + s->stats.incoming.framing_bytes++; p->frame_type = *cur; switch (p->frame_type) { case 0: @@ -184,9 +175,9 @@ grpc_error *grpc_chttp2_data_parser_parse( gpr_asprintf(&msg, "Bad GRPC frame type 0x%02x", p->frame_type); p->error = GRPC_ERROR_CREATE(msg); p->error = grpc_error_set_int(p->error, GRPC_ERROR_INT_STREAM_ID, - (intptr_t)stream_parsing->id); + (intptr_t)s->id); gpr_free(msg); - msg = gpr_dump_slice(slice, GPR_DUMP_HEX | GPR_DUMP_ASCII); + msg = grpc_dump_slice(slice, GPR_DUMP_HEX | GPR_DUMP_ASCII); p->error = grpc_error_set_str(p->error, GRPC_ERROR_STR_RAW_BYTES, msg); gpr_free(msg); @@ -201,7 +192,7 @@ grpc_error *grpc_chttp2_data_parser_parse( } /* fallthrough */ case GRPC_CHTTP2_DATA_FH_1: - stream_parsing->stats.incoming.framing_bytes++; + s->stats.incoming.framing_bytes++; p->frame_size = ((uint32_t)*cur) << 24; if (++cur == end) { p->state = GRPC_CHTTP2_DATA_FH_2; @@ -209,7 +200,7 @@ grpc_error *grpc_chttp2_data_parser_parse( } /* fallthrough */ case GRPC_CHTTP2_DATA_FH_2: - stream_parsing->stats.incoming.framing_bytes++; + s->stats.incoming.framing_bytes++; p->frame_size |= ((uint32_t)*cur) << 16; if (++cur == end) { p->state = GRPC_CHTTP2_DATA_FH_3; @@ -217,7 +208,7 @@ grpc_error *grpc_chttp2_data_parser_parse( } /* fallthrough */ case GRPC_CHTTP2_DATA_FH_3: - stream_parsing->stats.incoming.framing_bytes++; + s->stats.incoming.framing_bytes++; p->frame_size |= ((uint32_t)*cur) << 8; if (++cur == end) { p->state = GRPC_CHTTP2_DATA_FH_4; @@ -225,7 +216,7 @@ grpc_error *grpc_chttp2_data_parser_parse( } /* fallthrough */ case GRPC_CHTTP2_DATA_FH_4: - stream_parsing->stats.incoming.framing_bytes++; + s->stats.incoming.framing_bytes++; p->frame_size |= ((uint32_t)*cur); p->state = GRPC_CHTTP2_DATA_FRAME; ++cur; @@ -234,35 +225,32 @@ grpc_error *grpc_chttp2_data_parser_parse( message_flags |= GRPC_WRITE_INTERNAL_COMPRESS; } p->parsing_frame = incoming_byte_stream = - grpc_chttp2_incoming_byte_stream_create( - exec_ctx, transport_parsing, stream_parsing, p->frame_size, - message_flags, &p->incoming_frames); + grpc_chttp2_incoming_byte_stream_create(exec_ctx, t, s, p->frame_size, + message_flags); /* fallthrough */ case GRPC_CHTTP2_DATA_FRAME: - grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, - stream_parsing); if (cur == end) { return GRPC_ERROR_NONE; } uint32_t remaining = (uint32_t)(end - cur); if (remaining == p->frame_size) { - stream_parsing->stats.incoming.data_bytes += p->frame_size; + s->stats.incoming.data_bytes += p->frame_size; grpc_chttp2_incoming_byte_stream_push( exec_ctx, p->parsing_frame, - gpr_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg))); + grpc_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg))); grpc_chttp2_incoming_byte_stream_finished(exec_ctx, p->parsing_frame, - GRPC_ERROR_NONE, 1); + GRPC_ERROR_NONE); p->parsing_frame = NULL; p->state = GRPC_CHTTP2_DATA_FH_0; return GRPC_ERROR_NONE; } else if (remaining > p->frame_size) { - stream_parsing->stats.incoming.data_bytes += p->frame_size; + s->stats.incoming.data_bytes += p->frame_size; grpc_chttp2_incoming_byte_stream_push( exec_ctx, p->parsing_frame, - gpr_slice_sub(slice, (size_t)(cur - beg), - (size_t)(cur + p->frame_size - beg))); + grpc_slice_sub(slice, (size_t)(cur - beg), + (size_t)(cur + p->frame_size - beg))); grpc_chttp2_incoming_byte_stream_finished(exec_ctx, p->parsing_frame, - GRPC_ERROR_NONE, 1); + GRPC_ERROR_NONE); p->parsing_frame = NULL; cur += p->frame_size; goto fh_0; /* loop */ @@ -270,12 +258,27 @@ grpc_error *grpc_chttp2_data_parser_parse( GPR_ASSERT(remaining <= p->frame_size); grpc_chttp2_incoming_byte_stream_push( exec_ctx, p->parsing_frame, - gpr_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg))); + grpc_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg))); p->frame_size -= remaining; - stream_parsing->stats.incoming.data_bytes += remaining; + s->stats.incoming.data_bytes += remaining; return GRPC_ERROR_NONE; } } GPR_UNREACHABLE_CODE(return GRPC_ERROR_CREATE("Should never reach here")); } + +grpc_error *grpc_chttp2_data_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + grpc_slice slice, int is_last) { + grpc_chttp2_data_parser *p = parser; + grpc_error *error = parse_inner(exec_ctx, p, t, s, slice); + + if (is_last && p->is_last_frame) { + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, false, + GRPC_ERROR_NONE); + } + + return error; +} diff --git a/src/core/ext/transport/chttp2/transport/frame_data.h b/src/core/ext/transport/chttp2/transport/frame_data.h index a21a7942b9..264ad14608 100644 --- a/src/core/ext/transport/chttp2/transport/frame_data.h +++ b/src/core/ext/transport/chttp2/transport/frame_data.h @@ -36,8 +36,8 @@ /* Parser for GRPC streams embedded in DATA frames */ -#include <grpc/support/slice.h> -#include <grpc/support/slice_buffer.h> +#include <grpc/slice.h> +#include <grpc/slice_buffer.h> #include "src/core/ext/transport/chttp2/transport/frame.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/transport/byte_stream.h" @@ -69,7 +69,6 @@ typedef struct { grpc_error *error; int is_frame_compressed; - grpc_chttp2_incoming_frame_queue incoming_frames; grpc_chttp2_incoming_byte_stream *parsing_frame; } grpc_chttp2_data_parser; @@ -92,14 +91,14 @@ grpc_error *grpc_chttp2_data_parser_begin_frame(grpc_chttp2_data_parser *parser, /* handle a slice of a data frame - is_last indicates the last slice of a frame */ -grpc_error *grpc_chttp2_data_parser_parse( - grpc_exec_ctx *exec_ctx, void *parser, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last); +grpc_error *grpc_chttp2_data_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + grpc_slice slice, int is_last); -void grpc_chttp2_encode_data(uint32_t id, gpr_slice_buffer *inbuf, +void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer *inbuf, uint32_t write_bytes, int is_eof, grpc_transport_one_way_stats *stats, - gpr_slice_buffer *outbuf); + grpc_slice_buffer *outbuf); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_DATA_H */ diff --git a/src/core/ext/transport/chttp2/transport/frame_goaway.c b/src/core/ext/transport/chttp2/transport/frame_goaway.c index 299e27ad70..d99d486c1b 100644 --- a/src/core/ext/transport/chttp2/transport/frame_goaway.c +++ b/src/core/ext/transport/chttp2/transport/frame_goaway.c @@ -67,12 +67,13 @@ grpc_error *grpc_chttp2_goaway_parser_begin_frame(grpc_chttp2_goaway_parser *p, return GRPC_ERROR_NONE; } -grpc_error *grpc_chttp2_goaway_parser_parse( - grpc_exec_ctx *exec_ctx, void *parser, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) { - uint8_t *const beg = GPR_SLICE_START_PTR(slice); - uint8_t *const end = GPR_SLICE_END_PTR(slice); +grpc_error *grpc_chttp2_goaway_parser_parse(grpc_exec_ctx *exec_ctx, + void *parser, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + grpc_slice slice, int is_last) { + uint8_t *const beg = GRPC_SLICE_START_PTR(slice); + uint8_t *const end = GRPC_SLICE_END_PTR(slice); uint8_t *cur = beg; grpc_chttp2_goaway_parser *p = parser; @@ -148,12 +149,9 @@ grpc_error *grpc_chttp2_goaway_parser_parse( p->debug_pos += (uint32_t)(end - cur); p->state = GRPC_CHTTP2_GOAWAY_DEBUG; if (is_last) { - transport_parsing->goaway_received = 1; - transport_parsing->goaway_last_stream_index = p->last_stream_id; - gpr_slice_unref(transport_parsing->goaway_text); - transport_parsing->goaway_error = (grpc_status_code)p->error_code; - transport_parsing->goaway_text = - gpr_slice_new(p->debug_data, p->debug_length, gpr_free); + grpc_chttp2_add_incoming_goaway( + exec_ctx, t, (uint32_t)p->error_code, + grpc_slice_new(p->debug_data, p->debug_length, gpr_free)); p->debug_data = NULL; } return GRPC_ERROR_NONE; @@ -162,13 +160,13 @@ grpc_error *grpc_chttp2_goaway_parser_parse( } void grpc_chttp2_goaway_append(uint32_t last_stream_id, uint32_t error_code, - gpr_slice debug_data, - gpr_slice_buffer *slice_buffer) { - gpr_slice header = gpr_slice_malloc(9 + 4 + 4); - uint8_t *p = GPR_SLICE_START_PTR(header); + grpc_slice debug_data, + grpc_slice_buffer *slice_buffer) { + grpc_slice header = grpc_slice_malloc(9 + 4 + 4); + uint8_t *p = GRPC_SLICE_START_PTR(header); uint32_t frame_length; - GPR_ASSERT(GPR_SLICE_LENGTH(debug_data) < UINT32_MAX - 4 - 4); - frame_length = 4 + 4 + (uint32_t)GPR_SLICE_LENGTH(debug_data); + GPR_ASSERT(GRPC_SLICE_LENGTH(debug_data) < UINT32_MAX - 4 - 4); + frame_length = 4 + 4 + (uint32_t)GRPC_SLICE_LENGTH(debug_data); /* frame header: length */ *p++ = (uint8_t)(frame_length >> 16); @@ -193,7 +191,7 @@ void grpc_chttp2_goaway_append(uint32_t last_stream_id, uint32_t error_code, *p++ = (uint8_t)(error_code >> 16); *p++ = (uint8_t)(error_code >> 8); *p++ = (uint8_t)(error_code); - GPR_ASSERT(p == GPR_SLICE_END_PTR(header)); - gpr_slice_buffer_add(slice_buffer, header); - gpr_slice_buffer_add(slice_buffer, debug_data); + GPR_ASSERT(p == GRPC_SLICE_END_PTR(header)); + grpc_slice_buffer_add(slice_buffer, header); + grpc_slice_buffer_add(slice_buffer, debug_data); } diff --git a/src/core/ext/transport/chttp2/transport/frame_goaway.h b/src/core/ext/transport/chttp2/transport/frame_goaway.h index eb4303405a..21fe819488 100644 --- a/src/core/ext/transport/chttp2/transport/frame_goaway.h +++ b/src/core/ext/transport/chttp2/transport/frame_goaway.h @@ -34,9 +34,9 @@ #ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_GOAWAY_H #define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_GOAWAY_H +#include <grpc/slice.h> +#include <grpc/slice_buffer.h> #include <grpc/support/port_platform.h> -#include <grpc/support/slice.h> -#include <grpc/support/slice_buffer.h> #include "src/core/ext/transport/chttp2/transport/frame.h" #include "src/core/lib/iomgr/exec_ctx.h" @@ -65,13 +65,14 @@ void grpc_chttp2_goaway_parser_init(grpc_chttp2_goaway_parser *p); void grpc_chttp2_goaway_parser_destroy(grpc_chttp2_goaway_parser *p); grpc_error *grpc_chttp2_goaway_parser_begin_frame( grpc_chttp2_goaway_parser *parser, uint32_t length, uint8_t flags); -grpc_error *grpc_chttp2_goaway_parser_parse( - grpc_exec_ctx *exec_ctx, void *parser, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last); +grpc_error *grpc_chttp2_goaway_parser_parse(grpc_exec_ctx *exec_ctx, + void *parser, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + grpc_slice slice, int is_last); void grpc_chttp2_goaway_append(uint32_t last_stream_id, uint32_t error_code, - gpr_slice debug_data, - gpr_slice_buffer *slice_buffer); + grpc_slice debug_data, + grpc_slice_buffer *slice_buffer); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_GOAWAY_H */ diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c index 1f814ab1bd..7de5f6362d 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.c +++ b/src/core/ext/transport/chttp2/transport/frame_ping.c @@ -40,9 +40,9 @@ #include <grpc/support/log.h> #include <grpc/support/string_util.h> -gpr_slice grpc_chttp2_ping_create(uint8_t ack, uint8_t *opaque_8bytes) { - gpr_slice slice = gpr_slice_malloc(9 + 8); - uint8_t *p = GPR_SLICE_START_PTR(slice); +grpc_slice grpc_chttp2_ping_create(uint8_t ack, uint8_t *opaque_8bytes) { + grpc_slice slice = grpc_slice_malloc(9 + 8); + uint8_t *p = GRPC_SLICE_START_PTR(slice); *p++ = 0; *p++ = 0; @@ -73,12 +73,12 @@ grpc_error *grpc_chttp2_ping_parser_begin_frame(grpc_chttp2_ping_parser *parser, return GRPC_ERROR_NONE; } -grpc_error *grpc_chttp2_ping_parser_parse( - grpc_exec_ctx *exec_ctx, void *parser, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) { - uint8_t *const beg = GPR_SLICE_START_PTR(slice); - uint8_t *const end = GPR_SLICE_END_PTR(slice); +grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + grpc_slice slice, int is_last) { + uint8_t *const beg = GRPC_SLICE_START_PTR(slice); + uint8_t *const end = GRPC_SLICE_END_PTR(slice); uint8_t *cur = beg; grpc_chttp2_ping_parser *p = parser; @@ -91,10 +91,11 @@ grpc_error *grpc_chttp2_ping_parser_parse( if (p->byte == 8) { GPR_ASSERT(is_last); if (p->is_ack) { - grpc_chttp2_ack_ping(exec_ctx, transport_parsing, p->opaque_8bytes); + grpc_chttp2_ack_ping(exec_ctx, t, p->opaque_8bytes); } else { - gpr_slice_buffer_add(&transport_parsing->qbuf, - grpc_chttp2_ping_create(1, p->opaque_8bytes)); + grpc_slice_buffer_add(&t->qbuf, + grpc_chttp2_ping_create(1, p->opaque_8bytes)); + grpc_chttp2_initiate_write(exec_ctx, t, false, "ping response"); } } diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.h b/src/core/ext/transport/chttp2/transport/frame_ping.h index 5a8723421c..b9889e2d11 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.h +++ b/src/core/ext/transport/chttp2/transport/frame_ping.h @@ -34,7 +34,7 @@ #ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_PING_H #define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_PING_H -#include <grpc/support/slice.h> +#include <grpc/slice.h> #include "src/core/ext/transport/chttp2/transport/frame.h" #include "src/core/lib/iomgr/exec_ctx.h" @@ -44,13 +44,13 @@ typedef struct { uint8_t opaque_8bytes[8]; } grpc_chttp2_ping_parser; -gpr_slice grpc_chttp2_ping_create(uint8_t ack, uint8_t *opaque_8bytes); +grpc_slice grpc_chttp2_ping_create(uint8_t ack, uint8_t *opaque_8bytes); grpc_error *grpc_chttp2_ping_parser_begin_frame(grpc_chttp2_ping_parser *parser, uint32_t length, uint8_t flags); -grpc_error *grpc_chttp2_ping_parser_parse( - grpc_exec_ctx *exec_ctx, void *parser, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last); +grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + grpc_slice slice, int is_last); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_PING_H */ diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c index e3a3c9e4a7..b4c5ed769b 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c @@ -39,13 +39,15 @@ #include <grpc/support/string_util.h> #include "src/core/ext/transport/chttp2/transport/frame.h" +#include "src/core/ext/transport/chttp2/transport/http2_errors.h" +#include "src/core/ext/transport/chttp2/transport/status_conversion.h" -gpr_slice grpc_chttp2_rst_stream_create(uint32_t id, uint32_t code, - grpc_transport_one_way_stats *stats) { +grpc_slice grpc_chttp2_rst_stream_create(uint32_t id, uint32_t code, + grpc_transport_one_way_stats *stats) { static const size_t frame_size = 13; - gpr_slice slice = gpr_slice_malloc(frame_size); + grpc_slice slice = grpc_slice_malloc(frame_size); stats->framing_bytes += frame_size; - uint8_t *p = GPR_SLICE_START_PTR(slice); + uint8_t *p = GRPC_SLICE_START_PTR(slice); // Frame size. *p++ = 0; @@ -83,12 +85,13 @@ grpc_error *grpc_chttp2_rst_stream_parser_begin_frame( return GRPC_ERROR_NONE; } -grpc_error *grpc_chttp2_rst_stream_parser_parse( - grpc_exec_ctx *exec_ctx, void *parser, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) { - uint8_t *const beg = GPR_SLICE_START_PTR(slice); - uint8_t *const end = GPR_SLICE_END_PTR(slice); +grpc_error *grpc_chttp2_rst_stream_parser_parse(grpc_exec_ctx *exec_ctx, + void *parser, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + grpc_slice slice, int is_last) { + uint8_t *const beg = GRPC_SLICE_START_PTR(slice); + uint8_t *const end = GRPC_SLICE_END_PTR(slice); uint8_t *cur = beg; grpc_chttp2_rst_stream_parser *p = parser; @@ -97,19 +100,28 @@ grpc_error *grpc_chttp2_rst_stream_parser_parse( cur++; p->byte++; } - stream_parsing->stats.incoming.framing_bytes += (uint64_t)(end - cur); + s->stats.incoming.framing_bytes += (uint64_t)(end - cur); if (p->byte == 4) { GPR_ASSERT(is_last); - stream_parsing->received_close = 1; - if (stream_parsing->forced_close_error == GRPC_ERROR_NONE) { - stream_parsing->forced_close_error = grpc_error_set_int( - GRPC_ERROR_CREATE("RST_STREAM"), GRPC_ERROR_INT_HTTP2_ERROR, - (intptr_t)((((uint32_t)p->reason_bytes[0]) << 24) | - (((uint32_t)p->reason_bytes[1]) << 16) | - (((uint32_t)p->reason_bytes[2]) << 8) | - (((uint32_t)p->reason_bytes[3])))); + uint32_t reason = (((uint32_t)p->reason_bytes[0]) << 24) | + (((uint32_t)p->reason_bytes[1]) << 16) | + (((uint32_t)p->reason_bytes[2]) << 8) | + (((uint32_t)p->reason_bytes[3])); + grpc_error *error = GRPC_ERROR_NONE; + if (reason != GRPC_CHTTP2_NO_ERROR) { + error = grpc_error_set_int(GRPC_ERROR_CREATE("RST_STREAM"), + GRPC_ERROR_INT_HTTP2_ERROR, (intptr_t)reason); + grpc_status_code status_code = grpc_chttp2_http2_error_to_grpc_status( + (grpc_chttp2_error_code)reason, s->deadline); + char *status_details; + gpr_asprintf(&status_details, "Received RST_STREAM with error code %d", + reason); + grpc_slice slice_details = grpc_slice_from_copied_string(status_details); + gpr_free(status_details); + grpc_chttp2_fake_status(exec_ctx, t, s, status_code, &slice_details); } + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, error); } return GRPC_ERROR_NONE; diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.h b/src/core/ext/transport/chttp2/transport/frame_rst_stream.h index 11cf94f3ea..779507a617 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.h +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.h @@ -34,7 +34,7 @@ #ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_RST_STREAM_H #define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_RST_STREAM_H -#include <grpc/support/slice.h> +#include <grpc/slice.h> #include "src/core/ext/transport/chttp2/transport/frame.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/transport/transport.h" @@ -44,14 +44,15 @@ typedef struct { uint8_t reason_bytes[4]; } grpc_chttp2_rst_stream_parser; -gpr_slice grpc_chttp2_rst_stream_create(uint32_t stream_id, uint32_t code, - grpc_transport_one_way_stats *stats); +grpc_slice grpc_chttp2_rst_stream_create(uint32_t stream_id, uint32_t code, + grpc_transport_one_way_stats *stats); grpc_error *grpc_chttp2_rst_stream_parser_begin_frame( grpc_chttp2_rst_stream_parser *parser, uint32_t length, uint8_t flags); -grpc_error *grpc_chttp2_rst_stream_parser_parse( - grpc_exec_ctx *exec_ctx, void *parser, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last); +grpc_error *grpc_chttp2_rst_stream_parser_parse(grpc_exec_ctx *exec_ctx, + void *parser, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + grpc_slice slice, int is_last); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_RST_STREAM_H */ diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.c b/src/core/ext/transport/chttp2/transport/frame_settings.c index 04b96c4cd9..98facae87f 100644 --- a/src/core/ext/transport/chttp2/transport/frame_settings.c +++ b/src/core/ext/transport/chttp2/transport/frame_settings.c @@ -82,19 +82,19 @@ static uint8_t *fill_header(uint8_t *out, uint32_t length, uint8_t flags) { return out; } -gpr_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new, - uint32_t force_mask, size_t count) { +grpc_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new, + uint32_t force_mask, size_t count) { size_t i; uint32_t n = 0; - gpr_slice output; + grpc_slice output; uint8_t *p; for (i = 0; i < count; i++) { n += (new[i] != old[i] || (force_mask & (1u << i)) != 0); } - output = gpr_slice_malloc(9 + 6 * n); - p = fill_header(GPR_SLICE_START_PTR(output), 6 * n, 0); + output = grpc_slice_malloc(9 + 6 * n); + p = fill_header(GRPC_SLICE_START_PTR(output), 6 * n, 0); for (i = 0; i < count; i++) { if (new[i] != old[i] || (force_mask & (1u << i)) != 0) { @@ -109,14 +109,14 @@ gpr_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new, } } - GPR_ASSERT(p == GPR_SLICE_END_PTR(output)); + GPR_ASSERT(p == GRPC_SLICE_END_PTR(output)); return output; } -gpr_slice grpc_chttp2_settings_ack_create(void) { - gpr_slice output = gpr_slice_malloc(9); - fill_header(GPR_SLICE_START_PTR(output), 0, GRPC_CHTTP2_FLAG_ACK); +grpc_slice grpc_chttp2_settings_ack_create(void) { + grpc_slice output = grpc_slice_malloc(9); + fill_header(GRPC_SLICE_START_PTR(output), 0, GRPC_CHTTP2_FLAG_ACK); return output; } @@ -143,13 +143,13 @@ grpc_error *grpc_chttp2_settings_parser_begin_frame( } } -grpc_error *grpc_chttp2_settings_parser_parse( - grpc_exec_ctx *exec_ctx, void *p, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) { +grpc_error *grpc_chttp2_settings_parser_parse(grpc_exec_ctx *exec_ctx, void *p, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + grpc_slice slice, int is_last) { grpc_chttp2_settings_parser *parser = p; - const uint8_t *cur = GPR_SLICE_START_PTR(slice); - const uint8_t *end = GPR_SLICE_END_PTR(slice); + const uint8_t *cur = GRPC_SLICE_START_PTR(slice); + const uint8_t *end = GRPC_SLICE_END_PTR(slice); char *msg; if (parser->is_ack) { @@ -162,11 +162,9 @@ grpc_error *grpc_chttp2_settings_parser_parse( if (cur == end) { parser->state = GRPC_CHTTP2_SPS_ID0; if (is_last) { - transport_parsing->settings_updated = 1; memcpy(parser->target_settings, parser->incoming_settings, GRPC_CHTTP2_NUM_SETTINGS * sizeof(uint32_t)); - gpr_slice_buffer_add(&transport_parsing->qbuf, - grpc_chttp2_settings_ack_create()); + grpc_slice_buffer_add(&t->qbuf, grpc_chttp2_settings_ack_create()); } return GRPC_ERROR_NONE; } @@ -226,9 +224,9 @@ grpc_error *grpc_chttp2_settings_parser_parse( break; case GRPC_CHTTP2_DISCONNECT_ON_INVALID_VALUE: grpc_chttp2_goaway_append( - transport_parsing->last_incoming_stream_id, sp->error_value, - gpr_slice_from_static_string("HTTP2 settings error"), - &transport_parsing->qbuf); + t->last_new_stream_id, sp->error_value, + grpc_slice_from_static_string("HTTP2 settings error"), + &t->qbuf); gpr_asprintf(&msg, "invalid value %u passed for %s", parser->value, sp->name); grpc_error *err = GRPC_ERROR_CREATE(msg); @@ -238,18 +236,17 @@ grpc_error *grpc_chttp2_settings_parser_parse( } if (parser->id == GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE && parser->incoming_settings[parser->id] != parser->value) { - transport_parsing->initial_window_update = + t->initial_window_update = (int64_t)parser->value - parser->incoming_settings[parser->id]; if (grpc_http_trace) { gpr_log(GPR_DEBUG, "adding %d for initial_window change", - (int)transport_parsing->initial_window_update); + (int)t->initial_window_update); } } parser->incoming_settings[parser->id] = parser->value; if (grpc_http_trace) { gpr_log(GPR_DEBUG, "CHTTP2:%s: got setting %d = %d", - transport_parsing->is_client ? "CLI" : "SVR", parser->id, - parser->value); + t->is_client ? "CLI" : "SVR", parser->id, parser->value); } } else if (grpc_http_trace) { gpr_log(GPR_ERROR, "CHTTP2: Ignoring unknown setting %d (value %d)", diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.h b/src/core/ext/transport/chttp2/transport/frame_settings.h index f654c598c8..a29dc82106 100644 --- a/src/core/ext/transport/chttp2/transport/frame_settings.h +++ b/src/core/ext/transport/chttp2/transport/frame_settings.h @@ -34,8 +34,8 @@ #ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_SETTINGS_H #define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_SETTINGS_H +#include <grpc/slice.h> #include <grpc/support/port_platform.h> -#include <grpc/support/slice.h> #include "src/core/ext/transport/chttp2/transport/frame.h" #include "src/core/lib/iomgr/exec_ctx.h" @@ -87,17 +87,18 @@ extern const grpc_chttp2_setting_parameters grpc_chttp2_settings_parameters[GRPC_CHTTP2_NUM_SETTINGS]; /* Create a settings frame by diffing old & new, and updating old to be new */ -gpr_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new, - uint32_t force_mask, size_t count); +grpc_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new, + uint32_t force_mask, size_t count); /* Create an ack settings frame */ -gpr_slice grpc_chttp2_settings_ack_create(void); +grpc_slice grpc_chttp2_settings_ack_create(void); grpc_error *grpc_chttp2_settings_parser_begin_frame( grpc_chttp2_settings_parser *parser, uint32_t length, uint8_t flags, uint32_t *settings); -grpc_error *grpc_chttp2_settings_parser_parse( - grpc_exec_ctx *exec_ctx, void *parser, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last); +grpc_error *grpc_chttp2_settings_parser_parse(grpc_exec_ctx *exec_ctx, + void *parser, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + grpc_slice slice, int is_last); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_SETTINGS_H */ diff --git a/src/core/ext/transport/chttp2/transport/frame_window_update.c b/src/core/ext/transport/chttp2/transport/frame_window_update.c index 3cf848fd5c..31a31c2871 100644 --- a/src/core/ext/transport/chttp2/transport/frame_window_update.c +++ b/src/core/ext/transport/chttp2/transport/frame_window_update.c @@ -38,12 +38,12 @@ #include <grpc/support/log.h> #include <grpc/support/string_util.h> -gpr_slice grpc_chttp2_window_update_create( +grpc_slice grpc_chttp2_window_update_create( uint32_t id, uint32_t window_update, grpc_transport_one_way_stats *stats) { static const size_t frame_size = 13; - gpr_slice slice = gpr_slice_malloc(frame_size); + grpc_slice slice = grpc_slice_malloc(frame_size); stats->header_bytes += frame_size; - uint8_t *p = GPR_SLICE_START_PTR(slice); + uint8_t *p = GRPC_SLICE_START_PTR(slice); GPR_ASSERT(window_update); @@ -80,11 +80,10 @@ grpc_error *grpc_chttp2_window_update_parser_begin_frame( } grpc_error *grpc_chttp2_window_update_parser_parse( - grpc_exec_ctx *exec_ctx, void *parser, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) { - uint8_t *const beg = GPR_SLICE_START_PTR(slice); - uint8_t *const end = GPR_SLICE_END_PTR(slice); + grpc_exec_ctx *exec_ctx, void *parser, grpc_chttp2_transport *t, + grpc_chttp2_stream *s, grpc_slice slice, int is_last) { + uint8_t *const beg = GRPC_SLICE_START_PTR(slice); + uint8_t *const end = GRPC_SLICE_END_PTR(slice); uint8_t *cur = beg; grpc_chttp2_window_update_parser *p = parser; @@ -94,8 +93,8 @@ grpc_error *grpc_chttp2_window_update_parser_parse( p->byte++; } - if (stream_parsing != NULL) { - stream_parsing->stats.incoming.framing_bytes += (uint32_t)(end - cur); + if (s != NULL) { + s->stats.incoming.framing_bytes += (uint32_t)(end - cur); } if (p->byte == 4) { @@ -109,17 +108,26 @@ grpc_error *grpc_chttp2_window_update_parser_parse( } GPR_ASSERT(is_last); - if (transport_parsing->incoming_stream_id != 0) { - if (stream_parsing != NULL) { - GRPC_CHTTP2_FLOW_CREDIT_STREAM("parse", transport_parsing, - stream_parsing, outgoing_window, + if (t->incoming_stream_id != 0) { + if (s != NULL) { + bool was_zero = s->outgoing_window <= 0; + GRPC_CHTTP2_FLOW_CREDIT_STREAM("parse", t, s, outgoing_window, received_update); - grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, - stream_parsing); + bool is_zero = s->outgoing_window <= 0; + if (was_zero && !is_zero) { + grpc_chttp2_become_writable(exec_ctx, t, s, false, + "stream.read_flow_control"); + } } } else { - GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parse", transport_parsing, - outgoing_window, received_update); + bool was_zero = t->outgoing_window <= 0; + GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parse", t, outgoing_window, + received_update); + bool is_zero = t->outgoing_window <= 0; + if (was_zero && !is_zero) { + grpc_chttp2_initiate_write(exec_ctx, t, false, + "new_global_flow_control"); + } } } diff --git a/src/core/ext/transport/chttp2/transport/frame_window_update.h b/src/core/ext/transport/chttp2/transport/frame_window_update.h index 1bcbbf9247..f75dfb3d87 100644 --- a/src/core/ext/transport/chttp2/transport/frame_window_update.h +++ b/src/core/ext/transport/chttp2/transport/frame_window_update.h @@ -34,7 +34,7 @@ #ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_WINDOW_UPDATE_H #define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_WINDOW_UPDATE_H -#include <grpc/support/slice.h> +#include <grpc/slice.h> #include "src/core/ext/transport/chttp2/transport/frame.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/transport/transport.h" @@ -45,14 +45,13 @@ typedef struct { uint32_t amount; } grpc_chttp2_window_update_parser; -gpr_slice grpc_chttp2_window_update_create(uint32_t id, uint32_t window_delta, - grpc_transport_one_way_stats *stats); +grpc_slice grpc_chttp2_window_update_create( + uint32_t id, uint32_t window_delta, grpc_transport_one_way_stats *stats); grpc_error *grpc_chttp2_window_update_parser_begin_frame( grpc_chttp2_window_update_parser *parser, uint32_t length, uint8_t flags); grpc_error *grpc_chttp2_window_update_parser_parse( - grpc_exec_ctx *exec_ctx, void *parser, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last); + grpc_exec_ctx *exec_ctx, void *parser, grpc_chttp2_transport *t, + grpc_chttp2_stream *s, grpc_slice slice, int is_last); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_WINDOW_UPDATE_H */ diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 581471ba02..eb68fe3138 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -76,7 +76,7 @@ typedef struct { uint8_t seen_regular_header; /* output stream id */ uint32_t stream_id; - gpr_slice_buffer *output; + grpc_slice_buffer *output; grpc_transport_one_way_stats *stats; /* maximum size of a frame */ size_t max_frame_size; @@ -104,7 +104,7 @@ static void finish_frame(framer_state *st, int is_header_boundary, type = st->is_first_frame ? GRPC_CHTTP2_FRAME_HEADER : GRPC_CHTTP2_FRAME_CONTINUATION; fill_header( - GPR_SLICE_START_PTR(st->output->slices[st->header_idx]), type, + GRPC_SLICE_START_PTR(st->output->slices[st->header_idx]), type, st->stream_id, st->output->length - st->output_length_at_start_of_frame, (uint8_t)((is_last_in_stream ? GRPC_CHTTP2_DATA_FLAG_END_STREAM : 0) | (is_header_boundary ? GRPC_CHTTP2_DATA_FLAG_END_HEADERS : 0))); @@ -116,7 +116,7 @@ static void finish_frame(framer_state *st, int is_header_boundary, output before beginning */ static void begin_frame(framer_state *st) { st->header_idx = - gpr_slice_buffer_add_indexed(st->output, gpr_slice_malloc(9)); + grpc_slice_buffer_add_indexed(st->output, grpc_slice_malloc(9)); st->output_length_at_start_of_frame = st->output->length; } @@ -147,18 +147,18 @@ static void inc_filter(uint8_t idx, uint32_t *sum, uint8_t *elems) { } } -static void add_header_data(framer_state *st, gpr_slice slice) { - size_t len = GPR_SLICE_LENGTH(slice); +static void add_header_data(framer_state *st, grpc_slice slice) { + size_t len = GRPC_SLICE_LENGTH(slice); size_t remaining; if (len == 0) return; remaining = st->max_frame_size + st->output_length_at_start_of_frame - st->output->length; if (len <= remaining) { st->stats->header_bytes += len; - gpr_slice_buffer_add(st->output, slice); + grpc_slice_buffer_add(st->output, slice); } else { st->stats->header_bytes += remaining; - gpr_slice_buffer_add(st->output, gpr_slice_split_head(&slice, remaining)); + grpc_slice_buffer_add(st->output, grpc_slice_split_head(&slice, remaining)); finish_frame(st, 0, 0); begin_frame(st); add_header_data(st, slice); @@ -167,7 +167,7 @@ static void add_header_data(framer_state *st, gpr_slice slice) { static uint8_t *add_tiny_header_data(framer_state *st, size_t len) { ensure_space(st, len); - return gpr_slice_buffer_tiny_add(st->output, len); + return grpc_slice_buffer_tiny_add(st->output, len); } static void evict_entry(grpc_chttp2_hpack_compressor *c) { @@ -268,9 +268,10 @@ static void emit_indexed(grpc_chttp2_hpack_compressor *c, uint32_t elem_index, len); } -static gpr_slice get_wire_value(grpc_mdelem *elem, uint8_t *huffman_prefix) { - if (grpc_is_binary_header((const char *)GPR_SLICE_START_PTR(elem->key->slice), - GPR_SLICE_LENGTH(elem->key->slice))) { +static grpc_slice get_wire_value(grpc_mdelem *elem, uint8_t *huffman_prefix) { + if (grpc_is_binary_header( + (const char *)GRPC_SLICE_START_PTR(elem->key->slice), + GRPC_SLICE_LENGTH(elem->key->slice))) { *huffman_prefix = 0x80; return grpc_mdstr_as_base64_encoded_and_huffman_compressed(elem->value); } @@ -284,8 +285,8 @@ static void emit_lithdr_incidx(grpc_chttp2_hpack_compressor *c, framer_state *st) { uint32_t len_pfx = GRPC_CHTTP2_VARINT_LENGTH(key_index, 2); uint8_t huffman_prefix; - gpr_slice value_slice = get_wire_value(elem, &huffman_prefix); - size_t len_val = GPR_SLICE_LENGTH(value_slice); + grpc_slice value_slice = get_wire_value(elem, &huffman_prefix); + size_t len_val = GRPC_SLICE_LENGTH(value_slice); uint32_t len_val_len; GPR_ASSERT(len_val <= UINT32_MAX); len_val_len = GRPC_CHTTP2_VARINT_LENGTH((uint32_t)len_val, 1); @@ -293,7 +294,7 @@ static void emit_lithdr_incidx(grpc_chttp2_hpack_compressor *c, add_tiny_header_data(st, len_pfx), len_pfx); GRPC_CHTTP2_WRITE_VARINT((uint32_t)len_val, 1, huffman_prefix, add_tiny_header_data(st, len_val_len), len_val_len); - add_header_data(st, gpr_slice_ref(value_slice)); + add_header_data(st, grpc_slice_ref(value_slice)); } static void emit_lithdr_noidx(grpc_chttp2_hpack_compressor *c, @@ -301,8 +302,8 @@ static void emit_lithdr_noidx(grpc_chttp2_hpack_compressor *c, framer_state *st) { uint32_t len_pfx = GRPC_CHTTP2_VARINT_LENGTH(key_index, 4); uint8_t huffman_prefix; - gpr_slice value_slice = get_wire_value(elem, &huffman_prefix); - size_t len_val = GPR_SLICE_LENGTH(value_slice); + grpc_slice value_slice = get_wire_value(elem, &huffman_prefix); + size_t len_val = GRPC_SLICE_LENGTH(value_slice); uint32_t len_val_len; GPR_ASSERT(len_val <= UINT32_MAX); len_val_len = GRPC_CHTTP2_VARINT_LENGTH((uint32_t)len_val, 1); @@ -310,45 +311,45 @@ static void emit_lithdr_noidx(grpc_chttp2_hpack_compressor *c, add_tiny_header_data(st, len_pfx), len_pfx); GRPC_CHTTP2_WRITE_VARINT((uint32_t)len_val, 1, huffman_prefix, add_tiny_header_data(st, len_val_len), len_val_len); - add_header_data(st, gpr_slice_ref(value_slice)); + add_header_data(st, grpc_slice_ref(value_slice)); } static void emit_lithdr_incidx_v(grpc_chttp2_hpack_compressor *c, grpc_mdelem *elem, framer_state *st) { - uint32_t len_key = (uint32_t)GPR_SLICE_LENGTH(elem->key->slice); + uint32_t len_key = (uint32_t)GRPC_SLICE_LENGTH(elem->key->slice); uint8_t huffman_prefix; - gpr_slice value_slice = get_wire_value(elem, &huffman_prefix); - uint32_t len_val = (uint32_t)GPR_SLICE_LENGTH(value_slice); + grpc_slice value_slice = get_wire_value(elem, &huffman_prefix); + uint32_t len_val = (uint32_t)GRPC_SLICE_LENGTH(value_slice); uint32_t len_key_len = GRPC_CHTTP2_VARINT_LENGTH(len_key, 1); uint32_t len_val_len = GRPC_CHTTP2_VARINT_LENGTH(len_val, 1); GPR_ASSERT(len_key <= UINT32_MAX); - GPR_ASSERT(GPR_SLICE_LENGTH(value_slice) <= UINT32_MAX); + GPR_ASSERT(GRPC_SLICE_LENGTH(value_slice) <= UINT32_MAX); *add_tiny_header_data(st, 1) = 0x40; GRPC_CHTTP2_WRITE_VARINT(len_key, 1, 0x00, add_tiny_header_data(st, len_key_len), len_key_len); - add_header_data(st, gpr_slice_ref(elem->key->slice)); + add_header_data(st, grpc_slice_ref(elem->key->slice)); GRPC_CHTTP2_WRITE_VARINT(len_val, 1, huffman_prefix, add_tiny_header_data(st, len_val_len), len_val_len); - add_header_data(st, gpr_slice_ref(value_slice)); + add_header_data(st, grpc_slice_ref(value_slice)); } static void emit_lithdr_noidx_v(grpc_chttp2_hpack_compressor *c, grpc_mdelem *elem, framer_state *st) { - uint32_t len_key = (uint32_t)GPR_SLICE_LENGTH(elem->key->slice); + uint32_t len_key = (uint32_t)GRPC_SLICE_LENGTH(elem->key->slice); uint8_t huffman_prefix; - gpr_slice value_slice = get_wire_value(elem, &huffman_prefix); - uint32_t len_val = (uint32_t)GPR_SLICE_LENGTH(value_slice); + grpc_slice value_slice = get_wire_value(elem, &huffman_prefix); + uint32_t len_val = (uint32_t)GRPC_SLICE_LENGTH(value_slice); uint32_t len_key_len = GRPC_CHTTP2_VARINT_LENGTH(len_key, 1); uint32_t len_val_len = GRPC_CHTTP2_VARINT_LENGTH(len_val, 1); GPR_ASSERT(len_key <= UINT32_MAX); - GPR_ASSERT(GPR_SLICE_LENGTH(value_slice) <= UINT32_MAX); + GPR_ASSERT(GRPC_SLICE_LENGTH(value_slice) <= UINT32_MAX); *add_tiny_header_data(st, 1) = 0x00; GRPC_CHTTP2_WRITE_VARINT(len_key, 1, 0x00, add_tiny_header_data(st, len_key_len), len_key_len); - add_header_data(st, gpr_slice_ref(elem->key->slice)); + add_header_data(st, grpc_slice_ref(elem->key->slice)); GRPC_CHTTP2_WRITE_VARINT(len_val, 1, huffman_prefix, add_tiny_header_data(st, len_val_len), len_val_len); - add_header_data(st, gpr_slice_ref(value_slice)); + add_header_data(st, grpc_slice_ref(value_slice)); } static void emit_advertise_table_size_change(grpc_chttp2_hpack_compressor *c, @@ -373,8 +374,8 @@ static void hpack_enc(grpc_chttp2_hpack_compressor *c, grpc_mdelem *elem, uint32_t indices_key; int should_add_elem; - GPR_ASSERT(GPR_SLICE_LENGTH(elem->key->slice) > 0); - if (GPR_SLICE_START_PTR(elem->key->slice)[0] != ':') { /* regular header */ + GPR_ASSERT(GRPC_SLICE_LENGTH(elem->key->slice) > 0); + if (GRPC_SLICE_START_PTR(elem->key->slice)[0] != ':') { /* regular header */ st->seen_regular_header = 1; } else { GPR_ASSERT( @@ -546,7 +547,7 @@ void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor *c, grpc_metadata_batch *metadata, int is_eof, size_t max_frame_size, grpc_transport_one_way_stats *stats, - gpr_slice_buffer *outbuf) { + grpc_slice_buffer *outbuf) { framer_state st; grpc_linked_mdelem *l; gpr_timespec deadline; diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.h b/src/core/ext/transport/chttp2/transport/hpack_encoder.h index 4c3a931549..bcbd675ca2 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.h +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.h @@ -34,9 +34,9 @@ #ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_ENCODER_H #define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_ENCODER_H +#include <grpc/slice.h> +#include <grpc/slice_buffer.h> #include <grpc/support/port_platform.h> -#include <grpc/support/slice.h> -#include <grpc/support/slice_buffer.h> #include "src/core/ext/transport/chttp2/transport/frame.h" #include "src/core/lib/transport/metadata.h" #include "src/core/lib/transport/metadata_batch.h" @@ -93,6 +93,6 @@ void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor *c, uint32_t id, grpc_metadata_batch *metadata, int is_eof, size_t max_frame_size, grpc_transport_one_way_stats *stats, - gpr_slice_buffer *outbuf); + grpc_slice_buffer *outbuf); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_ENCODER_H */ diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c index 522455f7dc..6a9200b8db 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.c +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c @@ -50,6 +50,7 @@ #include <grpc/support/useful.h> #include "src/core/ext/transport/chttp2/transport/bin_encoder.h" +#include "src/core/ext/transport/chttp2/transport/http2_errors.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/support/string.h" @@ -78,69 +79,96 @@ typedef enum { a set of indirect jumps, and so not waste stack space. */ /* forward declarations for parsing states */ -static grpc_error *parse_begin(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_begin(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_error(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_error(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end, grpc_error *error); -static grpc_error *still_parse_error(grpc_chttp2_hpack_parser *p, +static grpc_error *still_parse_error(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_illegal_op(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_illegal_op(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_string_prefix(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_string_prefix(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_key_string(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_key_string(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); static grpc_error *parse_value_string_with_indexed_key( - grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); + grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, + const uint8_t *end); static grpc_error *parse_value_string_with_literal_key( - grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); + grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, + const uint8_t *end); -static grpc_error *parse_value0(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_value0(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_value1(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_value1(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_value2(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_value2(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_value3(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_value3(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_value4(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_value4(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_value5up(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_value5up(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_indexed_field(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_indexed_field(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_indexed_field_x(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_indexed_field_x(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_lithdr_incidx(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_incidx(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_lithdr_incidx_x(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_incidx_x(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_lithdr_incidx_v(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_incidx_v(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_lithdr_notidx(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_notidx(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_lithdr_notidx_x(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_notidx_x(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_lithdr_notidx_v(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_notidx_v(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_lithdr_nvridx(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_nvridx(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_lithdr_nvridx_x(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_nvridx_x(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_nvridx_v(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_max_tbl_size(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_max_tbl_size(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); -static grpc_error *parse_max_tbl_size_x(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_max_tbl_size_x(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end); /* we translate the first byte of a hpack field into one of these decoding @@ -639,8 +667,8 @@ static const uint8_t inverse_base64[256] = { }; /* emission helpers */ -static grpc_error *on_hdr(grpc_chttp2_hpack_parser *p, grpc_mdelem *md, - int add_to_table) { +static grpc_error *on_hdr(grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, + grpc_mdelem *md, int add_to_table) { if (add_to_table) { grpc_error *err = grpc_chttp2_hptbl_add(&p->table, md); if (err != GRPC_ERROR_NONE) return err; @@ -649,7 +677,7 @@ static grpc_error *on_hdr(grpc_chttp2_hpack_parser *p, grpc_mdelem *md, GRPC_MDELEM_UNREF(md); return GRPC_ERROR_CREATE("on_header callback not set"); } - p->on_header(p->on_header_user_data, md); + p->on_header(exec_ctx, p->on_header_user_data, md); return GRPC_ERROR_NONE; } @@ -661,78 +689,86 @@ static grpc_mdstr *take_string(grpc_chttp2_hpack_parser *p, } /* jump to the next state */ -static grpc_error *parse_next(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_next(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { p->state = *p->next_state++; - return p->state(p, cur, end); + return p->state(exec_ctx, p, cur, end); } /* begin parsing a header: all functionality is encoded into lookup tables above */ -static grpc_error *parse_begin(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_begin(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { if (cur == end) { p->state = parse_begin; return GRPC_ERROR_NONE; } - return first_byte_action[first_byte_lut[*cur]](p, cur, end); + return first_byte_action[first_byte_lut[*cur]](exec_ctx, p, cur, end); } /* stream dependency and prioritization data: we just skip it */ -static grpc_error *parse_stream_weight(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_stream_weight(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { if (cur == end) { p->state = parse_stream_weight; return GRPC_ERROR_NONE; } - return p->after_prioritization(p, cur + 1, end); + return p->after_prioritization(exec_ctx, p, cur + 1, end); } -static grpc_error *parse_stream_dep3(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_stream_dep3(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { if (cur == end) { p->state = parse_stream_dep3; return GRPC_ERROR_NONE; } - return parse_stream_weight(p, cur + 1, end); + return parse_stream_weight(exec_ctx, p, cur + 1, end); } -static grpc_error *parse_stream_dep2(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_stream_dep2(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { if (cur == end) { p->state = parse_stream_dep2; return GRPC_ERROR_NONE; } - return parse_stream_dep3(p, cur + 1, end); + return parse_stream_dep3(exec_ctx, p, cur + 1, end); } -static grpc_error *parse_stream_dep1(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_stream_dep1(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { if (cur == end) { p->state = parse_stream_dep1; return GRPC_ERROR_NONE; } - return parse_stream_dep2(p, cur + 1, end); + return parse_stream_dep2(exec_ctx, p, cur + 1, end); } -static grpc_error *parse_stream_dep0(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_stream_dep0(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { if (cur == end) { p->state = parse_stream_dep0; return GRPC_ERROR_NONE; } - return parse_stream_dep1(p, cur + 1, end); + return parse_stream_dep1(exec_ctx, p, cur + 1, end); } /* emit an indexed field; for now just logs it to console; jumps to begin the next field on completion */ -static grpc_error *finish_indexed_field(grpc_chttp2_hpack_parser *p, +static grpc_error *finish_indexed_field(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { grpc_mdelem *md = grpc_chttp2_hptbl_lookup(&p->table, p->index); @@ -743,21 +779,23 @@ static grpc_error *finish_indexed_field(grpc_chttp2_hpack_parser *p, GRPC_ERROR_INT_SIZE, (intptr_t)p->table.num_ents); } GRPC_MDELEM_REF(md); - grpc_error *err = on_hdr(p, md, 0); + grpc_error *err = on_hdr(exec_ctx, p, md, 0); if (err != GRPC_ERROR_NONE) return err; - return parse_begin(p, cur, end); + return parse_begin(exec_ctx, p, cur, end); } /* parse an indexed field with index < 127 */ -static grpc_error *parse_indexed_field(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_indexed_field(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { p->dynamic_table_update_allowed = 0; p->index = (*cur) & 0x7f; - return finish_indexed_field(p, cur + 1, end); + return finish_indexed_field(exec_ctx, p, cur + 1, end); } /* parse an indexed field with index >= 127 */ -static grpc_error *parse_indexed_field_x(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_indexed_field_x(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { static const grpc_chttp2_hpack_parser_state and_then[] = { @@ -766,49 +804,53 @@ static grpc_error *parse_indexed_field_x(grpc_chttp2_hpack_parser *p, p->next_state = and_then; p->index = 0x7f; p->parsing.value = &p->index; - return parse_value0(p, cur + 1, end); + return parse_value0(exec_ctx, p, cur + 1, end); } /* finish a literal header with incremental indexing: just log, and jump to ' begin */ -static grpc_error *finish_lithdr_incidx(grpc_chttp2_hpack_parser *p, +static grpc_error *finish_lithdr_incidx(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { grpc_mdelem *md = grpc_chttp2_hptbl_lookup(&p->table, p->index); GPR_ASSERT(md != NULL); /* handled in string parsing */ - grpc_error *err = - on_hdr(p, grpc_mdelem_from_metadata_strings(GRPC_MDSTR_REF(md->key), - take_string(p, &p->value)), - 1); - if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err); - return parse_begin(p, cur, end); + grpc_error *err = on_hdr( + exec_ctx, p, grpc_mdelem_from_metadata_strings(GRPC_MDSTR_REF(md->key), + take_string(p, &p->value)), + 1); + if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); + return parse_begin(exec_ctx, p, cur, end); } /* finish a literal header with incremental indexing with no index */ -static grpc_error *finish_lithdr_incidx_v(grpc_chttp2_hpack_parser *p, +static grpc_error *finish_lithdr_incidx_v(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { - grpc_error *err = - on_hdr(p, grpc_mdelem_from_metadata_strings(take_string(p, &p->key), - take_string(p, &p->value)), - 1); - if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err); - return parse_begin(p, cur, end); + grpc_error *err = on_hdr( + exec_ctx, p, grpc_mdelem_from_metadata_strings(take_string(p, &p->key), + take_string(p, &p->value)), + 1); + if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); + return parse_begin(exec_ctx, p, cur, end); } /* parse a literal header with incremental indexing; index < 63 */ -static grpc_error *parse_lithdr_incidx(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_incidx(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { static const grpc_chttp2_hpack_parser_state and_then[] = { parse_value_string_with_indexed_key, finish_lithdr_incidx}; p->dynamic_table_update_allowed = 0; p->next_state = and_then; p->index = (*cur) & 0x3f; - return parse_string_prefix(p, cur + 1, end); + return parse_string_prefix(exec_ctx, p, cur + 1, end); } /* parse a literal header with incremental indexing; index >= 63 */ -static grpc_error *parse_lithdr_incidx_x(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_incidx_x(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { static const grpc_chttp2_hpack_parser_state and_then[] = { @@ -818,11 +860,12 @@ static grpc_error *parse_lithdr_incidx_x(grpc_chttp2_hpack_parser *p, p->next_state = and_then; p->index = 0x3f; p->parsing.value = &p->index; - return parse_value0(p, cur + 1, end); + return parse_value0(exec_ctx, p, cur + 1, end); } /* parse a literal header with incremental indexing; index = 0 */ -static grpc_error *parse_lithdr_incidx_v(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_incidx_v(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { static const grpc_chttp2_hpack_parser_state and_then[] = { @@ -830,48 +873,52 @@ static grpc_error *parse_lithdr_incidx_v(grpc_chttp2_hpack_parser *p, parse_value_string_with_literal_key, finish_lithdr_incidx_v}; p->dynamic_table_update_allowed = 0; p->next_state = and_then; - return parse_string_prefix(p, cur + 1, end); + return parse_string_prefix(exec_ctx, p, cur + 1, end); } /* finish a literal header without incremental indexing */ -static grpc_error *finish_lithdr_notidx(grpc_chttp2_hpack_parser *p, +static grpc_error *finish_lithdr_notidx(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { grpc_mdelem *md = grpc_chttp2_hptbl_lookup(&p->table, p->index); GPR_ASSERT(md != NULL); /* handled in string parsing */ - grpc_error *err = - on_hdr(p, grpc_mdelem_from_metadata_strings(GRPC_MDSTR_REF(md->key), - take_string(p, &p->value)), - 0); - if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err); - return parse_begin(p, cur, end); + grpc_error *err = on_hdr( + exec_ctx, p, grpc_mdelem_from_metadata_strings(GRPC_MDSTR_REF(md->key), + take_string(p, &p->value)), + 0); + if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); + return parse_begin(exec_ctx, p, cur, end); } /* finish a literal header without incremental indexing with index = 0 */ -static grpc_error *finish_lithdr_notidx_v(grpc_chttp2_hpack_parser *p, +static grpc_error *finish_lithdr_notidx_v(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { - grpc_error *err = - on_hdr(p, grpc_mdelem_from_metadata_strings(take_string(p, &p->key), - take_string(p, &p->value)), - 0); - if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err); - return parse_begin(p, cur, end); + grpc_error *err = on_hdr( + exec_ctx, p, grpc_mdelem_from_metadata_strings(take_string(p, &p->key), + take_string(p, &p->value)), + 0); + if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); + return parse_begin(exec_ctx, p, cur, end); } /* parse a literal header without incremental indexing; index < 15 */ -static grpc_error *parse_lithdr_notidx(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_notidx(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { static const grpc_chttp2_hpack_parser_state and_then[] = { parse_value_string_with_indexed_key, finish_lithdr_notidx}; p->dynamic_table_update_allowed = 0; p->next_state = and_then; p->index = (*cur) & 0xf; - return parse_string_prefix(p, cur + 1, end); + return parse_string_prefix(exec_ctx, p, cur + 1, end); } /* parse a literal header without incremental indexing; index >= 15 */ -static grpc_error *parse_lithdr_notidx_x(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_notidx_x(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { static const grpc_chttp2_hpack_parser_state and_then[] = { @@ -881,11 +928,12 @@ static grpc_error *parse_lithdr_notidx_x(grpc_chttp2_hpack_parser *p, p->next_state = and_then; p->index = 0xf; p->parsing.value = &p->index; - return parse_value0(p, cur + 1, end); + return parse_value0(exec_ctx, p, cur + 1, end); } /* parse a literal header without incremental indexing; index == 0 */ -static grpc_error *parse_lithdr_notidx_v(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_notidx_v(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { static const grpc_chttp2_hpack_parser_state and_then[] = { @@ -893,48 +941,52 @@ static grpc_error *parse_lithdr_notidx_v(grpc_chttp2_hpack_parser *p, parse_value_string_with_literal_key, finish_lithdr_notidx_v}; p->dynamic_table_update_allowed = 0; p->next_state = and_then; - return parse_string_prefix(p, cur + 1, end); + return parse_string_prefix(exec_ctx, p, cur + 1, end); } /* finish a literal header that is never indexed */ -static grpc_error *finish_lithdr_nvridx(grpc_chttp2_hpack_parser *p, +static grpc_error *finish_lithdr_nvridx(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { grpc_mdelem *md = grpc_chttp2_hptbl_lookup(&p->table, p->index); GPR_ASSERT(md != NULL); /* handled in string parsing */ - grpc_error *err = - on_hdr(p, grpc_mdelem_from_metadata_strings(GRPC_MDSTR_REF(md->key), - take_string(p, &p->value)), - 0); - if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err); - return parse_begin(p, cur, end); + grpc_error *err = on_hdr( + exec_ctx, p, grpc_mdelem_from_metadata_strings(GRPC_MDSTR_REF(md->key), + take_string(p, &p->value)), + 0); + if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); + return parse_begin(exec_ctx, p, cur, end); } /* finish a literal header that is never indexed with an extra value */ -static grpc_error *finish_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p, +static grpc_error *finish_lithdr_nvridx_v(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { - grpc_error *err = - on_hdr(p, grpc_mdelem_from_metadata_strings(take_string(p, &p->key), - take_string(p, &p->value)), - 0); - if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err); - return parse_begin(p, cur, end); + grpc_error *err = on_hdr( + exec_ctx, p, grpc_mdelem_from_metadata_strings(take_string(p, &p->key), + take_string(p, &p->value)), + 0); + if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); + return parse_begin(exec_ctx, p, cur, end); } /* parse a literal header that is never indexed; index < 15 */ -static grpc_error *parse_lithdr_nvridx(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_nvridx(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { static const grpc_chttp2_hpack_parser_state and_then[] = { parse_value_string_with_indexed_key, finish_lithdr_nvridx}; p->dynamic_table_update_allowed = 0; p->next_state = and_then; p->index = (*cur) & 0xf; - return parse_string_prefix(p, cur + 1, end); + return parse_string_prefix(exec_ctx, p, cur + 1, end); } /* parse a literal header that is never indexed; index >= 15 */ -static grpc_error *parse_lithdr_nvridx_x(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_nvridx_x(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { static const grpc_chttp2_hpack_parser_state and_then[] = { @@ -944,11 +996,12 @@ static grpc_error *parse_lithdr_nvridx_x(grpc_chttp2_hpack_parser *p, p->next_state = and_then; p->index = 0xf; p->parsing.value = &p->index; - return parse_value0(p, cur + 1, end); + return parse_value0(exec_ctx, p, cur + 1, end); } /* parse a literal header that is never indexed; index == 0 */ -static grpc_error *parse_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_lithdr_nvridx_v(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { static const grpc_chttp2_hpack_parser_state and_then[] = { @@ -956,44 +1009,47 @@ static grpc_error *parse_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p, parse_value_string_with_literal_key, finish_lithdr_nvridx_v}; p->dynamic_table_update_allowed = 0; p->next_state = and_then; - return parse_string_prefix(p, cur + 1, end); + return parse_string_prefix(exec_ctx, p, cur + 1, end); } /* finish parsing a max table size change */ -static grpc_error *finish_max_tbl_size(grpc_chttp2_hpack_parser *p, +static grpc_error *finish_max_tbl_size(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { if (grpc_http_trace) { gpr_log(GPR_INFO, "MAX TABLE SIZE: %d", p->index); } grpc_error *err = grpc_chttp2_hptbl_set_current_table_size(&p->table, p->index); - if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err); - return parse_begin(p, cur, end); + if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); + return parse_begin(exec_ctx, p, cur, end); } /* parse a max table size change, max size < 15 */ -static grpc_error *parse_max_tbl_size(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_max_tbl_size(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { if (p->dynamic_table_update_allowed == 0) { return parse_error( - p, cur, end, + exec_ctx, p, cur, end, GRPC_ERROR_CREATE( "More than two max table size changes in a single frame")); } p->dynamic_table_update_allowed--; p->index = (*cur) & 0x1f; - return finish_max_tbl_size(p, cur + 1, end); + return finish_max_tbl_size(exec_ctx, p, cur + 1, end); } /* parse a max table size change, max size >= 15 */ -static grpc_error *parse_max_tbl_size_x(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_max_tbl_size_x(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { static const grpc_chttp2_hpack_parser_state and_then[] = { finish_max_tbl_size}; if (p->dynamic_table_update_allowed == 0) { return parse_error( - p, cur, end, + exec_ctx, p, cur, end, GRPC_ERROR_CREATE( "More than two max table size changes in a single frame")); } @@ -1001,11 +1057,12 @@ static grpc_error *parse_max_tbl_size_x(grpc_chttp2_hpack_parser *p, p->next_state = and_then; p->index = 0x1f; p->parsing.value = &p->index; - return parse_value0(p, cur + 1, end); + return parse_value0(exec_ctx, p, cur + 1, end); } /* a parse error: jam the parse state into parse_error, and return error */ -static grpc_error *parse_error(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_error(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end, grpc_error *err) { GPR_ASSERT(err != GRPC_ERROR_NONE); if (p->last_error == GRPC_ERROR_NONE) { @@ -1015,24 +1072,27 @@ static grpc_error *parse_error(grpc_chttp2_hpack_parser *p, const uint8_t *cur, return err; } -static grpc_error *still_parse_error(grpc_chttp2_hpack_parser *p, +static grpc_error *still_parse_error(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { return GRPC_ERROR_REF(p->last_error); } -static grpc_error *parse_illegal_op(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_illegal_op(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { GPR_ASSERT(cur != end); char *msg; gpr_asprintf(&msg, "Illegal hpack op code %d", *cur); grpc_error *err = GRPC_ERROR_CREATE(msg); gpr_free(msg); - return parse_error(p, cur, end, err); + return parse_error(exec_ctx, p, cur, end, err); } /* parse the 1st byte of a varint into p->parsing.value no overflow is possible */ -static grpc_error *parse_value0(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_value0(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { if (cur == end) { p->state = parse_value0; @@ -1042,15 +1102,16 @@ static grpc_error *parse_value0(grpc_chttp2_hpack_parser *p, const uint8_t *cur, *p->parsing.value += (*cur) & 0x7f; if ((*cur) & 0x80) { - return parse_value1(p, cur + 1, end); + return parse_value1(exec_ctx, p, cur + 1, end); } else { - return parse_next(p, cur + 1, end); + return parse_next(exec_ctx, p, cur + 1, end); } } /* parse the 2nd byte of a varint into p->parsing.value no overflow is possible */ -static grpc_error *parse_value1(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_value1(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { if (cur == end) { p->state = parse_value1; @@ -1060,15 +1121,16 @@ static grpc_error *parse_value1(grpc_chttp2_hpack_parser *p, const uint8_t *cur, *p->parsing.value += (((uint32_t)*cur) & 0x7f) << 7; if ((*cur) & 0x80) { - return parse_value2(p, cur + 1, end); + return parse_value2(exec_ctx, p, cur + 1, end); } else { - return parse_next(p, cur + 1, end); + return parse_next(exec_ctx, p, cur + 1, end); } } /* parse the 3rd byte of a varint into p->parsing.value no overflow is possible */ -static grpc_error *parse_value2(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_value2(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { if (cur == end) { p->state = parse_value2; @@ -1078,15 +1140,16 @@ static grpc_error *parse_value2(grpc_chttp2_hpack_parser *p, const uint8_t *cur, *p->parsing.value += (((uint32_t)*cur) & 0x7f) << 14; if ((*cur) & 0x80) { - return parse_value3(p, cur + 1, end); + return parse_value3(exec_ctx, p, cur + 1, end); } else { - return parse_next(p, cur + 1, end); + return parse_next(exec_ctx, p, cur + 1, end); } } /* parse the 4th byte of a varint into p->parsing.value no overflow is possible */ -static grpc_error *parse_value3(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_value3(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { if (cur == end) { p->state = parse_value3; @@ -1096,15 +1159,16 @@ static grpc_error *parse_value3(grpc_chttp2_hpack_parser *p, const uint8_t *cur, *p->parsing.value += (((uint32_t)*cur) & 0x7f) << 21; if ((*cur) & 0x80) { - return parse_value4(p, cur + 1, end); + return parse_value4(exec_ctx, p, cur + 1, end); } else { - return parse_next(p, cur + 1, end); + return parse_next(exec_ctx, p, cur + 1, end); } } /* parse the 5th byte of a varint into p->parsing.value depending on the byte, we may overflow, and care must be taken */ -static grpc_error *parse_value4(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_value4(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { uint8_t c; uint32_t cur_value; @@ -1130,9 +1194,9 @@ static grpc_error *parse_value4(grpc_chttp2_hpack_parser *p, const uint8_t *cur, *p->parsing.value = cur_value + add_value; if ((*cur) & 0x80) { - return parse_value5up(p, cur + 1, end); + return parse_value5up(exec_ctx, p, cur + 1, end); } else { - return parse_next(p, cur + 1, end); + return parse_next(exec_ctx, p, cur + 1, end); } error: @@ -1142,13 +1206,14 @@ error: *p->parsing.value, *cur); grpc_error *err = GRPC_ERROR_CREATE(msg); gpr_free(msg); - return parse_error(p, cur, end, err); + return parse_error(exec_ctx, p, cur, end, err); } /* parse any trailing bytes in a varint: it's possible to append an arbitrary number of 0x80's and not affect the value - a zero will terminate - and anything else will overflow */ -static grpc_error *parse_value5up(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_value5up(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { while (cur != end && *cur == 0x80) { ++cur; @@ -1160,7 +1225,7 @@ static grpc_error *parse_value5up(grpc_chttp2_hpack_parser *p, } if (*cur == 0) { - return parse_next(p, cur + 1, end); + return parse_next(exec_ctx, p, cur + 1, end); } char *msg; @@ -1170,11 +1235,12 @@ static grpc_error *parse_value5up(grpc_chttp2_hpack_parser *p, *p->parsing.value, *cur); grpc_error *err = GRPC_ERROR_CREATE(msg); gpr_free(msg); - return parse_error(p, cur, end, err); + return parse_error(exec_ctx, p, cur, end, err); } /* parse a string prefix */ -static grpc_error *parse_string_prefix(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_string_prefix(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { if (cur == end) { p->state = parse_string_prefix; @@ -1185,9 +1251,9 @@ static grpc_error *parse_string_prefix(grpc_chttp2_hpack_parser *p, p->huff = (*cur) >> 7; if (p->strlen == 0x7f) { p->parsing.value = &p->strlen; - return parse_value0(p, cur + 1, end); + return parse_value0(exec_ctx, p, cur + 1, end); } else { - return parse_next(p, cur + 1, end); + return parse_next(exec_ctx, p, cur + 1, end); } } @@ -1205,7 +1271,8 @@ static void append_bytes(grpc_chttp2_hpack_parser_string *str, str->length += (uint32_t)length; } -static grpc_error *append_string(grpc_chttp2_hpack_parser *p, +static grpc_error *append_string(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { grpc_chttp2_hpack_parser_string *str = p->parsing.str; uint32_t bits; @@ -1223,7 +1290,7 @@ static grpc_error *append_string(grpc_chttp2_hpack_parser *p, bits = inverse_base64[*cur]; ++cur; if (bits == 255) - return parse_error(p, cur, end, + return parse_error(exec_ctx, p, cur, end, GRPC_ERROR_CREATE("Illegal base64 character")); else if (bits == 64) goto b64_byte0; @@ -1238,7 +1305,7 @@ static grpc_error *append_string(grpc_chttp2_hpack_parser *p, bits = inverse_base64[*cur]; ++cur; if (bits == 255) - return parse_error(p, cur, end, + return parse_error(exec_ctx, p, cur, end, GRPC_ERROR_CREATE("Illegal base64 character")); else if (bits == 64) goto b64_byte1; @@ -1253,7 +1320,7 @@ static grpc_error *append_string(grpc_chttp2_hpack_parser *p, bits = inverse_base64[*cur]; ++cur; if (bits == 255) - return parse_error(p, cur, end, + return parse_error(exec_ctx, p, cur, end, GRPC_ERROR_CREATE("Illegal base64 character")); else if (bits == 64) goto b64_byte2; @@ -1268,7 +1335,7 @@ static grpc_error *append_string(grpc_chttp2_hpack_parser *p, bits = inverse_base64[*cur]; ++cur; if (bits == 255) - return parse_error(p, cur, end, + return parse_error(exec_ctx, p, cur, end, GRPC_ERROR_CREATE("Illegal base64 character")); else if (bits == 64) goto b64_byte3; @@ -1281,11 +1348,12 @@ static grpc_error *append_string(grpc_chttp2_hpack_parser *p, goto b64_byte0; } GPR_UNREACHABLE_CODE(return parse_error( - p, cur, end, GRPC_ERROR_CREATE("Should never reach here"))); + exec_ctx, p, cur, end, GRPC_ERROR_CREATE("Should never reach here"))); } /* append a null terminator to a string */ -static grpc_error *finish_str(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *finish_str(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { uint8_t terminator = 0; uint8_t decoded[2]; @@ -1298,7 +1366,7 @@ static grpc_error *finish_str(grpc_chttp2_hpack_parser *p, const uint8_t *cur, break; case B64_BYTE1: return parse_error( - p, cur, end, + exec_ctx, p, cur, end, GRPC_ERROR_CREATE("illegal base64 encoding")); /* illegal encoding */ case B64_BYTE2: bits = p->base64_buffer; @@ -1308,7 +1376,7 @@ static grpc_error *finish_str(grpc_chttp2_hpack_parser *p, const uint8_t *cur, bits & 0xffff); grpc_error *err = GRPC_ERROR_CREATE(msg); gpr_free(msg); - return parse_error(p, cur, end, err); + return parse_error(exec_ctx, p, cur, end, err); } decoded[0] = (uint8_t)(bits >> 16); append_bytes(str, decoded, 1); @@ -1321,7 +1389,7 @@ static grpc_error *finish_str(grpc_chttp2_hpack_parser *p, const uint8_t *cur, bits & 0xff); grpc_error *err = GRPC_ERROR_CREATE(msg); gpr_free(msg); - return parse_error(p, cur, end, err); + return parse_error(exec_ctx, p, cur, end, err); } decoded[0] = (uint8_t)(bits >> 16); decoded[1] = (uint8_t)(bits >> 8); @@ -1334,13 +1402,14 @@ static grpc_error *finish_str(grpc_chttp2_hpack_parser *p, const uint8_t *cur, } /* decode a nibble from a huffman encoded stream */ -static grpc_error *huff_nibble(grpc_chttp2_hpack_parser *p, uint8_t nibble) { +static grpc_error *huff_nibble(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, uint8_t nibble) { int16_t emit = emit_sub_tbl[16 * emit_tbl[p->huff_state] + nibble]; int16_t next = next_sub_tbl[16 * next_tbl[p->huff_state] + nibble]; if (emit != -1) { if (emit >= 0 && emit < 256) { uint8_t c = (uint8_t)emit; - grpc_error *err = append_string(p, &c, (&c) + 1); + grpc_error *err = append_string(exec_ctx, p, &c, (&c) + 1); if (err != GRPC_ERROR_NONE) return err; } else { assert(emit == 256); @@ -1351,42 +1420,45 @@ static grpc_error *huff_nibble(grpc_chttp2_hpack_parser *p, uint8_t nibble) { } /* decode full bytes from a huffman encoded stream */ -static grpc_error *add_huff_bytes(grpc_chttp2_hpack_parser *p, +static grpc_error *add_huff_bytes(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { for (; cur != end; ++cur) { - grpc_error *err = huff_nibble(p, *cur >> 4); - if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err); - err = huff_nibble(p, *cur & 0xf); - if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err); + grpc_error *err = huff_nibble(exec_ctx, p, *cur >> 4); + if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); + err = huff_nibble(exec_ctx, p, *cur & 0xf); + if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); } return GRPC_ERROR_NONE; } /* decode some string bytes based on the current decoding mode (huffman or not) */ -static grpc_error *add_str_bytes(grpc_chttp2_hpack_parser *p, +static grpc_error *add_str_bytes(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { if (p->huff) { - return add_huff_bytes(p, cur, end); + return add_huff_bytes(exec_ctx, p, cur, end); } else { - return append_string(p, cur, end); + return append_string(exec_ctx, p, cur, end); } } /* parse a string - tries to do large chunks at a time */ -static grpc_error *parse_string(grpc_chttp2_hpack_parser *p, const uint8_t *cur, +static grpc_error *parse_string(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { size_t remaining = p->strlen - p->strgot; size_t given = (size_t)(end - cur); if (remaining <= given) { - grpc_error *err = add_str_bytes(p, cur, cur + remaining); - if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err); - err = finish_str(p, cur + remaining, end); - if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err); - return parse_next(p, cur + remaining, end); + grpc_error *err = add_str_bytes(exec_ctx, p, cur, cur + remaining); + if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); + err = finish_str(exec_ctx, p, cur + remaining, end); + if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); + return parse_next(exec_ctx, p, cur + remaining, end); } else { - grpc_error *err = add_str_bytes(p, cur, cur + given); - if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err); + grpc_error *err = add_str_bytes(exec_ctx, p, cur, cur + given); + if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); GPR_ASSERT(given <= UINT32_MAX - p->strgot); p->strgot += (uint32_t)given; p->state = parse_string; @@ -1395,7 +1467,8 @@ static grpc_error *parse_string(grpc_chttp2_hpack_parser *p, const uint8_t *cur, } /* begin parsing a string - performs setup, calls parse_string */ -static grpc_error *begin_parse_string(grpc_chttp2_hpack_parser *p, +static grpc_error *begin_parse_string(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end, uint8_t binary, grpc_chttp2_hpack_parser_string *str) { @@ -1404,13 +1477,14 @@ static grpc_error *begin_parse_string(grpc_chttp2_hpack_parser *p, p->parsing.str = str; p->huff_state = 0; p->binary = binary; - return parse_string(p, cur, end); + return parse_string(exec_ctx, p, cur, end); } /* parse the key string */ -static grpc_error *parse_key_string(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_key_string(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { - return begin_parse_string(p, cur, end, NOT_BINARY, &p->key); + return begin_parse_string(exec_ctx, p, cur, end, NOT_BINARY, &p->key); } /* check if a key represents a binary header or not */ @@ -1428,31 +1502,34 @@ static grpc_error *is_binary_indexed_header(grpc_chttp2_hpack_parser *p, GRPC_ERROR_INT_INDEX, (intptr_t)p->index), GRPC_ERROR_INT_SIZE, (intptr_t)p->table.num_ents); } - *is = - grpc_is_binary_header((const char *)GPR_SLICE_START_PTR(elem->key->slice), - GPR_SLICE_LENGTH(elem->key->slice)); + *is = grpc_is_binary_header( + (const char *)GRPC_SLICE_START_PTR(elem->key->slice), + GRPC_SLICE_LENGTH(elem->key->slice)); return GRPC_ERROR_NONE; } /* parse the value string */ -static grpc_error *parse_value_string(grpc_chttp2_hpack_parser *p, +static grpc_error *parse_value_string(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end, bool is_binary) { - return begin_parse_string(p, cur, end, is_binary ? B64_BYTE0 : NOT_BINARY, - &p->value); + return begin_parse_string(exec_ctx, p, cur, end, + is_binary ? B64_BYTE0 : NOT_BINARY, &p->value); } static grpc_error *parse_value_string_with_indexed_key( - grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { + grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, + const uint8_t *end) { bool is_binary = false; grpc_error *err = is_binary_indexed_header(p, &is_binary); - if (err != GRPC_ERROR_NONE) return parse_error(p, cur, end, err); - return parse_value_string(p, cur, end, is_binary); + if (err != GRPC_ERROR_NONE) return parse_error(exec_ctx, p, cur, end, err); + return parse_value_string(exec_ctx, p, cur, end, is_binary); } static grpc_error *parse_value_string_with_literal_key( - grpc_chttp2_hpack_parser *p, const uint8_t *cur, const uint8_t *end) { - return parse_value_string(p, cur, end, is_binary_literal_header(p)); + grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *cur, + const uint8_t *end) { + return parse_value_string(exec_ctx, p, cur, end, is_binary_literal_header(p)); } /* PUBLIC INTERFACE */ @@ -1484,27 +1561,50 @@ void grpc_chttp2_hpack_parser_destroy(grpc_chttp2_hpack_parser *p) { gpr_free(p->value.str); } -grpc_error *grpc_chttp2_hpack_parser_parse(grpc_chttp2_hpack_parser *p, +grpc_error *grpc_chttp2_hpack_parser_parse(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *beg, const uint8_t *end) { /* TODO(ctiller): limit the distance of end from beg, and perform multiple steps in the event of a large chunk of data to limit stack space usage when no tail call optimization is available */ - return p->state(p, beg, end); + return p->state(exec_ctx, p, beg, end); } -grpc_error *grpc_chttp2_header_parser_parse( - grpc_exec_ctx *exec_ctx, void *hpack_parser, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last) { +typedef void (*maybe_complete_func_type)(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s); +static const maybe_complete_func_type maybe_complete_funcs[] = { + grpc_chttp2_maybe_complete_recv_initial_metadata, + grpc_chttp2_maybe_complete_recv_trailing_metadata}; + +static void force_client_rst_stream(grpc_exec_ctx *exec_ctx, void *sp, + grpc_error *error) { + grpc_chttp2_stream *s = sp; + grpc_chttp2_transport *t = s->t; + if (!s->write_closed) { + grpc_slice_buffer_add( + &t->qbuf, grpc_chttp2_rst_stream_create(s->id, GRPC_CHTTP2_NO_ERROR, + &s->stats.outgoing)); + grpc_chttp2_initiate_write(exec_ctx, t, false, "force_rst_stream"); + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, true, GRPC_ERROR_NONE); + } + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "final_rst"); +} + +grpc_error *grpc_chttp2_header_parser_parse(grpc_exec_ctx *exec_ctx, + void *hpack_parser, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + grpc_slice slice, int is_last) { grpc_chttp2_hpack_parser *parser = hpack_parser; GPR_TIMER_BEGIN("grpc_chttp2_hpack_parser_parse", 0); - if (stream_parsing != NULL) { - stream_parsing->stats.incoming.header_bytes += GPR_SLICE_LENGTH(slice); + if (s != NULL) { + s->stats.incoming.header_bytes += GRPC_SLICE_LENGTH(slice); } grpc_error *error = grpc_chttp2_hpack_parser_parse( - parser, GPR_SLICE_START_PTR(slice), GPR_SLICE_END_PTR(slice)); + exec_ctx, parser, GRPC_SLICE_START_PTR(slice), GRPC_SLICE_END_PTR(slice)); if (error != GRPC_ERROR_NONE) { GPR_TIMER_END("grpc_chttp2_hpack_parser_parse", 0); return error; @@ -1517,20 +1617,30 @@ grpc_error *grpc_chttp2_header_parser_parse( } /* need to check for null stream: this can occur if we receive an invalid stream id on a header */ - if (stream_parsing != NULL) { + if (s != NULL) { if (parser->is_boundary) { - if (stream_parsing->header_frames_received == - GPR_ARRAY_SIZE(stream_parsing->got_metadata_on_parse)) { + if (s->header_frames_received == GPR_ARRAY_SIZE(s->metadata_buffer)) { return GRPC_ERROR_CREATE("Too many trailer frames"); } - stream_parsing - ->got_metadata_on_parse[stream_parsing->header_frames_received] = 1; - stream_parsing->header_frames_received++; - grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, - stream_parsing); + s->published_metadata[s->header_frames_received] = + GRPC_METADATA_PUBLISHED_FROM_WIRE; + maybe_complete_funcs[s->header_frames_received](exec_ctx, t, s); + s->header_frames_received++; } if (parser->is_eof) { - stream_parsing->received_close = 1; + if (t->is_client && !s->write_closed) { + /* server eof ==> complete closure; we may need to forcefully close + the stream. Wait until the combiner lock is ready to be released + however -- it might be that we receive a RST_STREAM following this + and can avoid the extra write */ + GRPC_CHTTP2_STREAM_REF(s, "final_rst"); + grpc_combiner_execute_finally( + exec_ctx, t->combiner, + grpc_closure_create(force_client_rst_stream, s), GRPC_ERROR_NONE, + false); + } + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, false, + GRPC_ERROR_NONE); } } parser->on_header = NULL; diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.h b/src/core/ext/transport/chttp2/transport/hpack_parser.h index 78eb38db5e..a39bf466cd 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.h +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.h @@ -45,7 +45,8 @@ typedef struct grpc_chttp2_hpack_parser grpc_chttp2_hpack_parser; typedef grpc_error *(*grpc_chttp2_hpack_parser_state)( - grpc_chttp2_hpack_parser *p, const uint8_t *beg, const uint8_t *end); + grpc_exec_ctx *exec_ctx, grpc_chttp2_hpack_parser *p, const uint8_t *beg, + const uint8_t *end); typedef struct { char *str; @@ -55,7 +56,7 @@ typedef struct { struct grpc_chttp2_hpack_parser { /* user specified callback for each header output */ - void (*on_header)(void *user_data, grpc_mdelem *md); + void (*on_header)(grpc_exec_ctx *exec_ctx, void *user_data, grpc_mdelem *md); void *on_header_user_data; grpc_error *last_error; @@ -104,15 +105,17 @@ void grpc_chttp2_hpack_parser_destroy(grpc_chttp2_hpack_parser *p); void grpc_chttp2_hpack_parser_set_has_priority(grpc_chttp2_hpack_parser *p); /* returns 1 on success, 0 on error */ -grpc_error *grpc_chttp2_hpack_parser_parse(grpc_chttp2_hpack_parser *p, +grpc_error *grpc_chttp2_hpack_parser_parse(grpc_exec_ctx *exec_ctx, + grpc_chttp2_hpack_parser *p, const uint8_t *beg, const uint8_t *end); /* wraps grpc_chttp2_hpack_parser_parse to provide a frame level parser for the transport */ -grpc_error *grpc_chttp2_header_parser_parse( - grpc_exec_ctx *exec_ctx, void *hpack_parser, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, gpr_slice slice, int is_last); +grpc_error *grpc_chttp2_header_parser_parse(grpc_exec_ctx *exec_ctx, + void *hpack_parser, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + grpc_slice slice, int is_last); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_PARSER_H */ diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.c b/src/core/ext/transport/chttp2/transport/hpack_table.c index 2b73ec969e..2dc793d304 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_table.c +++ b/src/core/ext/transport/chttp2/transport/hpack_table.c @@ -226,8 +226,8 @@ grpc_mdelem *grpc_chttp2_hptbl_lookup(const grpc_chttp2_hptbl *tbl, /* Evict one element from the table */ static void evict1(grpc_chttp2_hptbl *tbl) { grpc_mdelem *first_ent = tbl->ents[tbl->first_ent]; - size_t elem_bytes = GPR_SLICE_LENGTH(first_ent->key->slice) + - GPR_SLICE_LENGTH(first_ent->value->slice) + + size_t elem_bytes = GRPC_SLICE_LENGTH(first_ent->key->slice) + + GRPC_SLICE_LENGTH(first_ent->value->slice) + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; GPR_ASSERT(elem_bytes <= tbl->mem_used); tbl->mem_used -= (uint32_t)elem_bytes; @@ -298,8 +298,8 @@ grpc_error *grpc_chttp2_hptbl_set_current_table_size(grpc_chttp2_hptbl *tbl, grpc_error *grpc_chttp2_hptbl_add(grpc_chttp2_hptbl *tbl, grpc_mdelem *md) { /* determine how many bytes of buffer this entry represents */ - size_t elem_bytes = GPR_SLICE_LENGTH(md->key->slice) + - GPR_SLICE_LENGTH(md->value->slice) + + size_t elem_bytes = GRPC_SLICE_LENGTH(md->key->slice) + + GRPC_SLICE_LENGTH(md->value->slice) + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; if (tbl->current_table_bytes > tbl->max_bytes) { diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.h b/src/core/ext/transport/chttp2/transport/hpack_table.h index 45bd9255bf..2ca130e64b 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_table.h +++ b/src/core/ext/transport/chttp2/transport/hpack_table.h @@ -34,8 +34,8 @@ #ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_TABLE_H #define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_TABLE_H +#include <grpc/slice.h> #include <grpc/support/port_platform.h> -#include <grpc/support/slice.h> #include "src/core/lib/iomgr/error.h" #include "src/core/lib/transport/metadata.h" diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 04b788b702..b74233d992 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -53,31 +53,25 @@ #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/transport_impl.h" -typedef struct grpc_chttp2_transport grpc_chttp2_transport; -typedef struct grpc_chttp2_stream grpc_chttp2_stream; - /* streams are kept in various linked lists depending on what things need to happen to them... this enum labels each list */ typedef enum { - GRPC_CHTTP2_LIST_ALL_STREAMS, - GRPC_CHTTP2_LIST_CHECK_READ_OPS, - GRPC_CHTTP2_LIST_UNANNOUNCED_INCOMING_WINDOW_AVAILABLE, GRPC_CHTTP2_LIST_WRITABLE, GRPC_CHTTP2_LIST_WRITING, - GRPC_CHTTP2_LIST_WRITTEN, - GRPC_CHTTP2_LIST_PARSING_SEEN, - GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_PARSING, - GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_WRITING, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT, - /* streams waiting for the outgoing window in the writing path, they will be - * merged to the stalled list or writable list under transport lock. */ - GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT, /** streams that are waiting to start because there are too many concurrent streams on the connection */ GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY, STREAM_LIST_COUNT /* must be last */ } grpc_chttp2_stream_list_id; +typedef enum { + GRPC_CHTTP2_WRITE_STATE_IDLE, + GRPC_CHTTP2_WRITE_STATE_WRITING, + GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE, + GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE_AND_COVERED_BY_POLLER, +} grpc_chttp2_write_state; + /* deframer state for the overall http2 stream of bytes */ typedef enum { /* prefix: one entry per http2 connection prefix byte */ @@ -144,6 +138,12 @@ typedef enum { GRPC_NUM_SETTING_SETS } grpc_chttp2_setting_set; +typedef enum { + GRPC_CHTTP2_NO_GOAWAY_SEND, + GRPC_CHTTP2_GOAWAY_SEND_SCHEDULED, + GRPC_CHTTP2_GOAWAY_SENT, +} grpc_chttp2_sent_goaway_state; + /* Outstanding ping request data */ typedef struct grpc_chttp2_outstanding_ping { uint8_t id[8]; @@ -152,6 +152,12 @@ typedef struct grpc_chttp2_outstanding_ping { struct grpc_chttp2_outstanding_ping *prev; } grpc_chttp2_outstanding_ping; +typedef struct grpc_chttp2_write_cb { + int64_t call_at_byte; + grpc_closure *closure; + struct grpc_chttp2_write_cb *next; +} grpc_chttp2_write_cb; + /* forward declared in frame_data.h */ struct grpc_chttp2_incoming_byte_stream { grpc_byte_stream base; @@ -161,16 +167,17 @@ struct grpc_chttp2_incoming_byte_stream { grpc_chttp2_transport *transport; grpc_chttp2_stream *stream; - int is_tail; + bool is_tail; gpr_mu slice_mu; // protects slices, on_next - gpr_slice_buffer slices; + grpc_slice_buffer slices; grpc_closure *on_next; - gpr_slice *next; + grpc_slice *next; + uint32_t remaining_bytes; struct { grpc_closure closure; - gpr_slice *slice; + grpc_slice *slice; size_t max_size_hint; grpc_closure *on_complete; } next_action; @@ -178,12 +185,68 @@ struct grpc_chttp2_incoming_byte_stream { grpc_closure finished_action; }; -typedef struct { - /** data to write next write */ - gpr_slice_buffer qbuf; +struct grpc_chttp2_transport { + grpc_transport base; /* must be first */ + gpr_refcount refs; + grpc_endpoint *ep; + char *peer_string; - /** window available for us to send to peer */ + grpc_combiner *combiner; + + /** write execution state of the transport */ + grpc_chttp2_write_state write_state; + + /** is the transport destroying itself? */ + uint8_t destroying; + /** has the upper layer closed the transport? */ + uint8_t closed; + + /** is there a read request to the endpoint outstanding? */ + uint8_t endpoint_reading; + + /** various lists of streams */ + grpc_chttp2_stream_list lists[STREAM_LIST_COUNT]; + + /** maps stream id to grpc_chttp2_stream objects */ + grpc_chttp2_stream_map stream_map; + + grpc_closure write_action_begin_locked; + grpc_closure write_action; + grpc_closure write_action_end; + grpc_closure write_action_end_locked; + + grpc_closure read_action_begin; + grpc_closure read_action_locked; + + /** incoming read bytes */ + grpc_slice_buffer read_buffer; + + /** address to place a newly accepted stream - set and unset by + grpc_chttp2_parsing_accept_stream; used by init_stream to + publish the accepted server stream */ + grpc_chttp2_stream **accepting_stream; + + struct { + /* accept stream callback */ + void (*accept_stream)(grpc_exec_ctx *exec_ctx, void *user_data, + grpc_transport *transport, const void *server_data); + void *accept_stream_user_data; + + /** connectivity tracking */ + grpc_connectivity_state_tracker state_tracker; + } channel_callback; + + /** data to write now */ + grpc_slice_buffer outbuf; + /** hpack encoding */ + grpc_chttp2_hpack_compressor hpack_compressor; int64_t outgoing_window; + /** is this a client? */ + uint8_t is_client; + + /** data to write next write */ + grpc_slice_buffer qbuf; + /** window available to announce to peer */ int64_t announce_incoming_window; /** how much window would we like to have for incoming_window */ @@ -192,10 +255,8 @@ typedef struct { /** have we seen a goaway */ uint8_t seen_goaway; /** have we sent a goaway */ - uint8_t sent_goaway; + grpc_chttp2_sent_goaway_state sent_goaway_state; - /** is this transport a client? */ - uint8_t is_client; /** are the local settings dirty and need to be sent? */ uint8_t dirtied_local_settings; /** have local settings been sent? */ @@ -212,49 +273,14 @@ typedef struct { /** how far to lookahead in a stream? */ uint32_t stream_lookahead; - /** last received stream id */ - uint32_t last_incoming_stream_id; + /** last new stream id */ + uint32_t last_new_stream_id; /** pings awaiting responses */ grpc_chttp2_outstanding_ping pings; /** next payload for an outgoing ping */ uint64_t ping_counter; - /** concurrent stream count: updated when not parsing, - so this is a strict over-estimation on the client */ - uint32_t concurrent_stream_count; -} grpc_chttp2_transport_global; - -typedef struct { - /** data to write now */ - gpr_slice_buffer outbuf; - /** hpack encoding */ - grpc_chttp2_hpack_compressor hpack_compressor; - int64_t outgoing_window; - /** is this a client? */ - uint8_t is_client; - /** callback for when writing is done */ - grpc_closure done_cb; - /** maximum frame size */ - uint32_t max_frame_size; -} grpc_chttp2_transport_writing; - -struct grpc_chttp2_transport_parsing { - /** is this transport a client? (boolean) */ - uint8_t is_client; - - /** were settings updated? */ - uint8_t settings_updated; - /** was a settings ack received? */ - uint8_t settings_ack_received; - /** was a goaway frame received? */ - uint8_t goaway_received; - - /** initial window change */ - int64_t initial_window_update; - - /** data to write later - after parsing */ - gpr_slice_buffer qbuf; /** parser for headers */ grpc_chttp2_hpack_parser hpack_parser; /** simple one shot parsers */ @@ -267,13 +293,12 @@ struct grpc_chttp2_transport_parsing { /** parser for goaway frames */ grpc_chttp2_goaway_parser goaway_parser; + /** initial window change */ + int64_t initial_window_update; + /** window available for peer to send to us */ int64_t incoming_window; - /** next stream id available at the time of beginning parsing */ - uint32_t next_stream_id; - uint32_t last_incoming_stream_id; - /* deframing */ grpc_chttp2_deframe_transport_state deframe_state; uint8_t incoming_frame_type; @@ -284,135 +309,54 @@ struct grpc_chttp2_transport_parsing { uint32_t incoming_frame_size; uint32_t incoming_stream_id; - /* current max frame size */ - uint32_t max_frame_size; - /* active parser */ void *parser_data; - grpc_chttp2_stream_parsing *incoming_stream; + grpc_chttp2_stream *incoming_stream; grpc_error *(*parser)(grpc_exec_ctx *exec_ctx, void *parser_user_data, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, - gpr_slice slice, int is_last); - - /* received settings */ - uint32_t settings[GRPC_CHTTP2_NUM_SETTINGS]; - /* last settings that were sent */ - uint32_t last_sent_settings[GRPC_CHTTP2_NUM_SETTINGS]; + grpc_chttp2_transport *t, grpc_chttp2_stream *s, + grpc_slice slice, int is_last); /* goaway data */ grpc_status_code goaway_error; uint32_t goaway_last_stream_index; - gpr_slice goaway_text; - - int64_t outgoing_window; + grpc_slice goaway_text; + + grpc_chttp2_write_cb *write_cb_pool; + + /* if non-NULL, close the transport with this error when writes are finished + */ + grpc_error *close_transport_on_writes_finished; + + /* buffer pool state */ + /** have we scheduled a benign cleanup? */ + bool benign_reclaimer_registered; + /** have we scheduled a destructive cleanup? */ + bool destructive_reclaimer_registered; + /** benign cleanup closure */ + grpc_closure benign_reclaimer; + grpc_closure benign_reclaimer_locked; + /** destructive cleanup closure */ + grpc_closure destructive_reclaimer; + grpc_closure destructive_reclaimer_locked; }; typedef enum { - /** no writing activity allowed */ - GRPC_CHTTP2_WRITES_CORKED, - /** no writing activity */ - GRPC_CHTTP2_WRITING_INACTIVE, - /** write has been requested and scheduled against the workqueue */ - GRPC_CHTTP2_WRITE_SCHEDULED, - /** write has been initiated after being reaped from the workqueue */ - GRPC_CHTTP2_WRITING, - /** write has been initiated, AND another write needs to be started once it's - done */ - GRPC_CHTTP2_WRITING_STALE_WITH_POLLER, - GRPC_CHTTP2_WRITING_STALE_NO_POLLER, -} grpc_chttp2_write_state; - -struct grpc_chttp2_transport { - grpc_transport base; /* must be first */ - gpr_refcount refs; - grpc_endpoint *ep; - char *peer_string; - - /** when this drops to zero it's safe to shutdown the endpoint */ - gpr_refcount shutdown_ep_refs; + GRPC_METADATA_NOT_PUBLISHED, + GRPC_METADATA_SYNTHESIZED_FROM_FAKE, + GRPC_METADATA_PUBLISHED_FROM_WIRE, + GPRC_METADATA_PUBLISHED_AT_CLOSE +} grpc_published_metadata_method; - struct { - grpc_combiner *combiner; - - /** is a thread currently in the global lock */ - bool global_active; - /** is a thread currently parsing */ - bool parsing_active; - /** write execution state of the transport */ - grpc_chttp2_write_state write_state; - /** has a check_read_ops been scheduled */ - bool check_read_ops_scheduled; - } executor; - - /** is the transport destroying itself? */ - uint8_t destroying; - /** has the upper layer closed the transport? */ - uint8_t closed; - - /** is there a read request to the endpoint outstanding? */ - uint8_t endpoint_reading; - - /** various lists of streams */ - grpc_chttp2_stream_list lists[STREAM_LIST_COUNT]; - - /** global state for reading/writing */ - grpc_chttp2_transport_global global; - /** state only accessible by the chain of execution that - set writing_state >= GRPC_WRITING, and only by the writing closure - chain. */ - grpc_chttp2_transport_writing writing; - /** state only accessible by the chain of execution that - set parsing_active=1 */ - grpc_chttp2_transport_parsing parsing; - - /** maps stream id to grpc_chttp2_stream objects; - owned by the parsing thread when parsing */ - grpc_chttp2_stream_map parsing_stream_map; - - /** streams created by the client (possibly during parsing); - merged with parsing_stream_map during unlock when no - parsing is occurring */ - grpc_chttp2_stream_map new_stream_map; - - /** closure to execute writing */ - grpc_closure writing_action; - /** closure to start reading from the endpoint */ - grpc_closure reading_action; - grpc_closure reading_action_locked; - grpc_closure post_parse_locked; - /** closure to actually do parsing */ - grpc_closure parsing_action; - /** closure to initiate writing */ - grpc_closure initiate_writing; - /** closure to finish writing */ - grpc_closure terminate_writing; - /** closure to flush read state up the stack */ - grpc_closure initiate_read_flush_locked; - - /** incoming read bytes */ - gpr_slice_buffer read_buffer; - - /** address to place a newly accepted stream - set and unset by - grpc_chttp2_parsing_accept_stream; used by init_stream to - publish the accepted server stream */ - grpc_chttp2_stream **accepting_stream; - - struct { - /* accept stream callback */ - void (*accept_stream)(grpc_exec_ctx *exec_ctx, void *user_data, - grpc_transport *transport, const void *server_data); - void *accept_stream_user_data; +struct grpc_chttp2_stream { + grpc_chttp2_transport *t; + grpc_stream_refcount *refcount; - /** connectivity tracking */ - grpc_connectivity_state_tracker state_tracker; - } channel_callback; + grpc_closure destroy_stream; + void *destroy_stream_arg; - /** Transport op to be applied post-parsing */ - grpc_transport_op *post_parsing_op; -}; + grpc_chttp2_stream_link links[STREAM_LIST_COUNT]; + uint8_t included[STREAM_LIST_COUNT]; -typedef struct { /** HTTP2 stream id for this stream, or zero if one has not been assigned */ uint32_t id; @@ -422,20 +366,22 @@ typedef struct { As the upper layer offers more bytes, this value increases. As bytes are read, this value decreases. */ uint32_t max_recv_bytes; - /** The number of bytes the upper layer has offered to read but we have - not yet announced to HTTP2 flow control. - As the upper layers offer to read more bytes, this value increases. - As we advertise incoming flow control window, this value decreases. */ - uint32_t unannounced_incoming_window_for_parse; - uint32_t unannounced_incoming_window_for_writing; /** things the upper layers would like to send */ grpc_metadata_batch *send_initial_metadata; grpc_closure *send_initial_metadata_finished; - grpc_byte_stream *send_message; - grpc_closure *send_message_finished; grpc_metadata_batch *send_trailing_metadata; grpc_closure *send_trailing_metadata_finished; + grpc_byte_stream *fetching_send_message; + uint32_t fetched_send_message_length; + grpc_slice fetching_slice; + int64_t next_message_end_offset; + int64_t flow_controlled_bytes_written; + bool complete_fetch_covered_by_poller; + grpc_closure complete_fetch; + grpc_closure complete_fetch_locked; + grpc_closure *fetching_send_message_finished; + grpc_metadata_batch *recv_initial_metadata; grpc_closure *recv_initial_metadata_ready; grpc_byte_stream **recv_message; @@ -455,95 +401,44 @@ typedef struct { bool read_closed; /** Are all published incoming byte streams closed. */ bool all_incoming_byte_streams_finished; - /** Is this stream in the stream map. */ - bool in_stream_map; /** Has this stream seen an error. If true, then pending incoming frames can be thrown away. */ bool seen_error; - bool exceeded_metadata_size; /** the error that resulted in this stream being read-closed */ grpc_error *read_closed_error; /** the error that resulted in this stream being write-closed */ grpc_error *write_closed_error; - bool published_initial_metadata; - bool published_trailing_metadata; + grpc_published_metadata_method published_metadata[2]; bool final_metadata_requested; - grpc_chttp2_incoming_metadata_buffer received_initial_metadata; - grpc_chttp2_incoming_metadata_buffer received_trailing_metadata; + grpc_chttp2_incoming_metadata_buffer metadata_buffer[2]; grpc_chttp2_incoming_frame_queue incoming_frames; gpr_timespec deadline; -} grpc_chttp2_stream_global; -typedef struct { - /** HTTP2 stream id for this stream, or zero if one has not been assigned */ - uint32_t id; - uint8_t fetching; - bool sent_initial_metadata; - uint8_t sent_message; - uint8_t sent_trailing_metadata; - uint8_t read_closed; - /** send this initial metadata */ - grpc_metadata_batch *send_initial_metadata; - grpc_byte_stream *send_message; - grpc_metadata_batch *send_trailing_metadata; - int64_t outgoing_window; - /** how much window should we announce? */ - uint32_t announce_window; - gpr_slice_buffer flow_controlled_buffer; - gpr_slice fetching_slice; - size_t stream_fetched; - grpc_closure finished_fetch; - /** stats gathered during the write */ - grpc_transport_one_way_stats stats; -} grpc_chttp2_stream_writing; - -struct grpc_chttp2_stream_parsing { /** saw some stream level error */ grpc_error *forced_close_error; - /** HTTP2 stream id for this stream, or zero if one has not been assigned */ - uint32_t id; - /** has this stream received a close */ - uint8_t received_close; /** how many header frames have we received? */ uint8_t header_frames_received; - /** which metadata did we get (on this parse) */ - uint8_t got_metadata_on_parse[2]; - /** should we raise the seen_error flag in transport_global */ - bool seen_error; - bool exceeded_metadata_size; /** window available for peer to send to us */ int64_t incoming_window; /** parsing state for data frames */ grpc_chttp2_data_parser data_parser; - /** amount of window given */ - int64_t outgoing_window; /** number of bytes received - reset at end of parse thread execution */ int64_t received_bytes; - /** stats gathered during the parse */ - grpc_transport_stream_stats stats; - /** incoming metadata */ - grpc_chttp2_incoming_metadata_buffer metadata_buffer[2]; -}; - -struct grpc_chttp2_stream { - grpc_chttp2_transport *t; - grpc_stream_refcount *refcount; - grpc_chttp2_stream_global global; - grpc_chttp2_stream_writing writing; - grpc_chttp2_stream_parsing parsing; - - grpc_closure init_stream; - grpc_closure destroy_stream; - void *destroy_stream_arg; + bool sent_initial_metadata; + bool sent_trailing_metadata; + /** how much window should we announce? */ + uint32_t announce_window; + grpc_slice_buffer flow_controlled_buffer; - grpc_chttp2_stream_link links[STREAM_LIST_COUNT]; - uint8_t included[STREAM_LIST_COUNT]; + grpc_chttp2_write_cb *on_write_finished_cbs; + grpc_chttp2_write_cb *finish_after_write; + size_t sending_bytes; }; /** Transport writing call flow: @@ -559,162 +454,72 @@ struct grpc_chttp2_stream { The actual call chain is documented in the implementation of this function. */ void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global, + grpc_chttp2_transport *t, bool covered_by_poller, const char *reason); /** Someone is unlocking the transport mutex: check to see if writes - are required, and schedule them if so */ -int grpc_chttp2_unlocking_check_writes(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *global, - grpc_chttp2_transport_writing *writing); -void grpc_chttp2_perform_writes( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing, - grpc_endpoint *endpoint); -void grpc_chttp2_terminate_writing(grpc_exec_ctx *exec_ctx, - void *transport_writing, grpc_error *error); -void grpc_chttp2_cleanup_writing(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *global, - grpc_chttp2_transport_writing *writing); - -void grpc_chttp2_prepare_to_read(grpc_chttp2_transport_global *global, - grpc_chttp2_transport_parsing *parsing); + are required, and frame them if so */ +bool grpc_chttp2_begin_write(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t); +void grpc_chttp2_end_write(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_error *error); + /** Process one slice of incoming data; return 1 if the connection is still viable after reading, or 0 if the connection should be torn down */ -grpc_error *grpc_chttp2_perform_read( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, - gpr_slice slice); -void grpc_chttp2_publish_reads(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *global, - grpc_chttp2_transport_parsing *parsing); - -bool grpc_chttp2_list_add_writable_stream( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global); +grpc_error *grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_slice slice); + +bool grpc_chttp2_list_add_writable_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream *s); /** Get a writable stream returns non-zero if there was a stream available */ -int grpc_chttp2_list_pop_writable_stream( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_global **stream_global, - grpc_chttp2_stream_writing **stream_writing); +int grpc_chttp2_list_pop_writable_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream **s); bool grpc_chttp2_list_remove_writable_stream( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) GRPC_MUST_USE_RESULT; - -void grpc_chttp2_list_add_writing_stream( - grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_writing *stream_writing); -int grpc_chttp2_list_have_writing_streams( - grpc_chttp2_transport_writing *transport_writing); -int grpc_chttp2_list_pop_writing_stream( - grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_writing **stream_writing); - -void grpc_chttp2_list_add_written_stream( - grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_writing *stream_writing); -int grpc_chttp2_list_pop_written_stream( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_global **stream_global, - grpc_chttp2_stream_writing **stream_writing); - -void grpc_chttp2_list_add_parsing_seen_stream( - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing); -int grpc_chttp2_list_pop_parsing_seen_stream( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_global **stream_global, - grpc_chttp2_stream_parsing **stream_parsing); - -void grpc_chttp2_list_add_waiting_for_concurrency( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global); -int grpc_chttp2_list_pop_waiting_for_concurrency( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global **stream_global); - -void grpc_chttp2_list_add_check_read_ops( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global); -bool grpc_chttp2_list_remove_check_read_ops( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global); -int grpc_chttp2_list_pop_check_read_ops( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global **stream_global); - -void grpc_chttp2_list_add_writing_stalled_by_transport( - grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_writing *stream_writing); -bool grpc_chttp2_list_flush_writing_stalled_by_transport( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing); - -void grpc_chttp2_list_add_stalled_by_transport( - grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_writing *stream_writing); -int grpc_chttp2_list_pop_stalled_by_transport( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global **stream_global); -void grpc_chttp2_list_remove_stalled_by_transport( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global); - -void grpc_chttp2_list_add_unannounced_incoming_window_available( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global); -void grpc_chttp2_list_remove_unannounced_incoming_window_available( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global); -int grpc_chttp2_list_pop_unannounced_incoming_window_available( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_global **stream_global, - grpc_chttp2_stream_parsing **stream_parsing); - -void grpc_chttp2_list_add_closed_waiting_for_parsing( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global); -int grpc_chttp2_list_pop_closed_waiting_for_parsing( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global **stream_global); - -void grpc_chttp2_list_add_closed_waiting_for_writing( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global); -int grpc_chttp2_list_pop_closed_waiting_for_writing( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global **stream_global); - -grpc_chttp2_stream_parsing *grpc_chttp2_parsing_lookup_stream( - grpc_chttp2_transport_parsing *transport_parsing, uint32_t id); -grpc_chttp2_stream_parsing *grpc_chttp2_parsing_accept_stream( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, - uint32_t id); - -void grpc_chttp2_add_incoming_goaway( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - uint32_t goaway_error, gpr_slice goaway_text); - -void grpc_chttp2_register_stream(grpc_chttp2_transport *t, - grpc_chttp2_stream *s); -/* returns 1 if this is the last stream, 0 otherwise */ -int grpc_chttp2_unregister_stream(grpc_chttp2_transport *t, - grpc_chttp2_stream *s) GRPC_MUST_USE_RESULT; -int grpc_chttp2_has_streams(grpc_chttp2_transport *t); -void grpc_chttp2_for_all_streams( - grpc_chttp2_transport_global *transport_global, void *user_data, - void (*cb)(grpc_chttp2_transport_global *transport_global, void *user_data, - grpc_chttp2_stream_global *stream_global)); - -void grpc_chttp2_parsing_become_skip_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing); - -void grpc_chttp2_complete_closure_step( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, grpc_closure **pclosure, - grpc_error *error); + grpc_chttp2_transport *t, grpc_chttp2_stream *s) GRPC_MUST_USE_RESULT; + +bool grpc_chttp2_list_add_writing_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream *s); +int grpc_chttp2_list_have_writing_streams(grpc_chttp2_transport *t); +int grpc_chttp2_list_pop_writing_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream **s); + +void grpc_chttp2_list_add_written_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream *s); +int grpc_chttp2_list_pop_written_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream **s); + +void grpc_chttp2_list_add_waiting_for_concurrency(grpc_chttp2_transport *t, + grpc_chttp2_stream *s); +int grpc_chttp2_list_pop_waiting_for_concurrency(grpc_chttp2_transport *t, + grpc_chttp2_stream **s); + +void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport *t, + grpc_chttp2_stream *s); +int grpc_chttp2_list_pop_stalled_by_transport(grpc_chttp2_transport *t, + grpc_chttp2_stream **s); +void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport *t, + grpc_chttp2_stream *s); + +grpc_chttp2_stream *grpc_chttp2_parsing_lookup_stream(grpc_chttp2_transport *t, + uint32_t id); +grpc_chttp2_stream *grpc_chttp2_parsing_accept_stream(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + uint32_t id); + +void grpc_chttp2_add_incoming_goaway(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + uint32_t goaway_error, + grpc_slice goaway_text); + +void grpc_chttp2_parsing_become_skip_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); + +void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, + grpc_closure **pclosure, + grpc_error *error, const char *desc); #define GRPC_CHTTP2_CLIENT_CONNECT_STRING "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" #define GRPC_CHTTP2_CLIENT_CONNECT_STRLEN \ @@ -805,57 +610,83 @@ void grpc_chttp2_flowctl_trace(const char *file, int line, const char *phase, const char *var2, int is_client, uint32_t stream_id, int64_t val1, int64_t val2); -void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream, - grpc_status_code status, gpr_slice *details); -void grpc_chttp2_mark_stream_closed( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, int close_reads, int close_writes, - grpc_error *error); +void grpc_chttp2_fake_status(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_stream *stream, + grpc_status_code status, grpc_slice *details); +void grpc_chttp2_mark_stream_closed(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, int close_reads, + int close_writes, grpc_error *error); void grpc_chttp2_start_writing(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global); + grpc_chttp2_transport *t); #ifdef GRPC_STREAM_REFCOUNT_DEBUG -#define GRPC_CHTTP2_STREAM_REF(stream_global, reason) \ - grpc_chttp2_stream_ref(stream_global, reason) -#define GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, reason) \ - grpc_chttp2_stream_unref(exec_ctx, stream_global, reason) -void grpc_chttp2_stream_ref(grpc_chttp2_stream_global *stream_global, - const char *reason); -void grpc_chttp2_stream_unref(grpc_exec_ctx *exec_ctx, - grpc_chttp2_stream_global *stream_global, +#define GRPC_CHTTP2_STREAM_REF(stream, reason) \ + grpc_chttp2_stream_ref(stream, reason) +#define GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream, reason) \ + grpc_chttp2_stream_unref(exec_ctx, stream, reason) +void grpc_chttp2_stream_ref(grpc_chttp2_stream *s, const char *reason); +void grpc_chttp2_stream_unref(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream *s, const char *reason); #else -#define GRPC_CHTTP2_STREAM_REF(stream_global, reason) \ - grpc_chttp2_stream_ref(stream_global) -#define GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, reason) \ - grpc_chttp2_stream_unref(exec_ctx, stream_global) -void grpc_chttp2_stream_ref(grpc_chttp2_stream_global *stream_global); -void grpc_chttp2_stream_unref(grpc_exec_ctx *exec_ctx, - grpc_chttp2_stream_global *stream_global); +#define GRPC_CHTTP2_STREAM_REF(stream, reason) grpc_chttp2_stream_ref(stream) +#define GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream, reason) \ + grpc_chttp2_stream_unref(exec_ctx, stream) +void grpc_chttp2_stream_ref(grpc_chttp2_stream *s); +void grpc_chttp2_stream_unref(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream *s); +#endif + +//#define GRPC_CHTTP2_REFCOUNTING_DEBUG 1 +#ifdef GRPC_CHTTP2_REFCOUNTING_DEBUG +#define GRPC_CHTTP2_REF_TRANSPORT(t, r) \ + grpc_chttp2_ref_transport(t, r, __FILE__, __LINE__) +#define GRPC_CHTTP2_UNREF_TRANSPORT(cl, t, r) \ + grpc_chttp2_unref_transport(cl, t, r, __FILE__, __LINE__) +void grpc_chttp2_unref_transport(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, const char *reason, + const char *file, int line); +void grpc_chttp2_ref_transport(grpc_chttp2_transport *t, const char *reason, + const char *file, int line); +#else +#define GRPC_CHTTP2_REF_TRANSPORT(t, r) grpc_chttp2_ref_transport(t) +#define GRPC_CHTTP2_UNREF_TRANSPORT(cl, t, r) grpc_chttp2_unref_transport(cl, t) +void grpc_chttp2_unref_transport(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); +void grpc_chttp2_ref_transport(grpc_chttp2_transport *t); #endif grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, uint32_t frame_size, - uint32_t flags, grpc_chttp2_incoming_frame_queue *add_to_queue); + grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, + uint32_t frame_size, uint32_t flags); void grpc_chttp2_incoming_byte_stream_push(grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, - gpr_slice slice); + grpc_slice slice); void grpc_chttp2_incoming_byte_stream_finished( grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, - grpc_error *error, int from_parsing_thread); + grpc_error *error); -void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_parsing *parsing, +void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, const uint8_t *opaque_8bytes); /** add a ref to the stream and add it to the writable list; ref will be dropped in writing.c */ void grpc_chttp2_become_writable(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global, - bool covered_by_poller, const char *reason); + grpc_chttp2_transport *t, + grpc_chttp2_stream *s, bool covered_by_poller, + const char *reason); + +void grpc_chttp2_cancel_stream(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, grpc_chttp2_stream *s, + grpc_error *due_to_error); + +void grpc_chttp2_maybe_complete_recv_initial_metadata(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s); +void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s); +void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_INTERNAL_H */ diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index 0e6d579ba9..b9c405158f 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -45,235 +45,42 @@ #include "src/core/lib/transport/static_metadata.h" #include "src/core/lib/transport/timeout_encoding.h" -#define TRANSPORT_FROM_PARSING(tp) \ - ((grpc_chttp2_transport *)((char *)(tp)-offsetof(grpc_chttp2_transport, \ - parsing))) - -static grpc_error *init_frame_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing); -static grpc_error *init_header_frame_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, - int is_continuation); -static grpc_error *init_data_frame_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing); -static grpc_error *init_rst_stream_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing); -static grpc_error *init_settings_frame_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing); -static grpc_error *init_window_update_frame_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing); -static grpc_error *init_ping_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing); -static grpc_error *init_goaway_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing); -static grpc_error *init_skip_frame_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, - int is_header); - -static grpc_error *parse_frame_slice( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, - gpr_slice slice, int is_last); - -void grpc_chttp2_prepare_to_read( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_parsing *transport_parsing) { - grpc_chttp2_stream_global *stream_global; - grpc_chttp2_stream_parsing *stream_parsing; - - GPR_TIMER_BEGIN("grpc_chttp2_prepare_to_read", 0); - - transport_parsing->next_stream_id = transport_global->next_stream_id; - memcpy(transport_parsing->last_sent_settings, - transport_global->settings[GRPC_SENT_SETTINGS], - sizeof(transport_parsing->last_sent_settings)); - transport_parsing->max_frame_size = - transport_global->settings[GRPC_ACKED_SETTINGS] - [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]; - - /* update the parsing view of incoming window */ - while (grpc_chttp2_list_pop_unannounced_incoming_window_available( - transport_global, transport_parsing, &stream_global, &stream_parsing)) { - GRPC_CHTTP2_FLOW_MOVE_STREAM("parse", transport_parsing, stream_parsing, - incoming_window, stream_global, - unannounced_incoming_window_for_parse); - } - - GPR_TIMER_END("grpc_chttp2_prepare_to_read", 0); -} - -void grpc_chttp2_publish_reads( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_parsing *transport_parsing) { - grpc_chttp2_stream_global *stream_global; - grpc_chttp2_stream_parsing *stream_parsing; - int was_zero; - int is_zero; - - /* transport_parsing->last_incoming_stream_id is used as - last-grpc_chttp2_stream-id when - sending GOAWAY frame. - https://tools.ietf.org/html/draft-ietf-httpbis-http2-17#section-6.8 - says that last-grpc_chttp2_stream-id is peer-initiated grpc_chttp2_stream - ID. So, - since we don't have server pushed streams, client should send - GOAWAY last-grpc_chttp2_stream-id=0 in this case. */ - if (!transport_parsing->is_client) { - transport_global->last_incoming_stream_id = - transport_parsing->last_incoming_stream_id; - } - - /* update global settings */ - if (transport_parsing->settings_updated) { - memcpy(transport_global->settings[GRPC_PEER_SETTINGS], - transport_parsing->settings, sizeof(transport_parsing->settings)); - transport_parsing->settings_updated = 0; - } - - /* update settings based on ack if received */ - if (transport_parsing->settings_ack_received) { - memcpy(transport_global->settings[GRPC_ACKED_SETTINGS], - transport_global->settings[GRPC_SENT_SETTINGS], - GRPC_CHTTP2_NUM_SETTINGS * sizeof(uint32_t)); - transport_parsing->settings_ack_received = 0; - transport_global->sent_local_settings = 0; - } - - /* move goaway to the global state if we received one (it will be - published later */ - if (transport_parsing->goaway_received) { - grpc_chttp2_add_incoming_goaway(exec_ctx, transport_global, - (uint32_t)transport_parsing->goaway_error, - transport_parsing->goaway_text); - transport_parsing->goaway_text = gpr_empty_slice(); - transport_parsing->goaway_received = 0; - } - - /* propagate flow control tokens to global state */ - was_zero = transport_global->outgoing_window <= 0; - GRPC_CHTTP2_FLOW_MOVE_TRANSPORT("parsed", transport_global, outgoing_window, - transport_parsing, outgoing_window); - is_zero = transport_global->outgoing_window <= 0; - if (was_zero && !is_zero) { - grpc_chttp2_initiate_write(exec_ctx, transport_global, false, - "new_global_flow_control"); - } - - if (transport_parsing->incoming_window < - transport_global->connection_window_target * 3 / 4) { - int64_t announce_bytes = transport_global->connection_window_target - - transport_parsing->incoming_window; - GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parsed", transport_global, - announce_incoming_window, announce_bytes); - GRPC_CHTTP2_FLOW_CREDIT_TRANSPORT("parsed", transport_parsing, - incoming_window, announce_bytes); - grpc_chttp2_initiate_write(exec_ctx, transport_global, false, - "global incoming window"); - } - - /* for each stream that saw an update, fixup global state */ - while (grpc_chttp2_list_pop_parsing_seen_stream( - transport_global, transport_parsing, &stream_global, &stream_parsing)) { - if (stream_parsing->seen_error) { - stream_global->seen_error = true; - stream_global->exceeded_metadata_size = - stream_parsing->exceeded_metadata_size; - grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global, - stream_global); - } - - /* flush stats to global stream state */ - grpc_transport_move_stats(&stream_parsing->stats, &stream_global->stats); - - /* update outgoing flow control window */ - was_zero = stream_global->outgoing_window <= 0; - GRPC_CHTTP2_FLOW_MOVE_STREAM("parsed", transport_global, stream_global, - outgoing_window, stream_parsing, - outgoing_window); - is_zero = stream_global->outgoing_window <= 0; - if (was_zero && !is_zero) { - grpc_chttp2_become_writable(exec_ctx, transport_global, stream_global, - false, "stream.read_flow_control"); - } - - stream_global->max_recv_bytes -= (uint32_t)GPR_MIN( - stream_global->max_recv_bytes, stream_parsing->received_bytes); - stream_parsing->received_bytes = 0; - - /* publish incoming stream ops */ - if (stream_global->incoming_frames.tail != NULL) { - stream_global->incoming_frames.tail->is_tail = 0; - } - if (stream_parsing->data_parser.incoming_frames.head != NULL) { - grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global, - stream_global); - } - grpc_chttp2_incoming_frame_queue_merge( - &stream_global->incoming_frames, - &stream_parsing->data_parser.incoming_frames); - if (stream_global->incoming_frames.tail != NULL) { - stream_global->incoming_frames.tail->is_tail = 1; - } - - if (!stream_global->published_initial_metadata && - stream_parsing->got_metadata_on_parse[0]) { - stream_parsing->got_metadata_on_parse[0] = 0; - stream_global->published_initial_metadata = 1; - GPR_SWAP(grpc_chttp2_incoming_metadata_buffer, - stream_parsing->metadata_buffer[0], - stream_global->received_initial_metadata); - grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global, - stream_global); - } - if (!stream_global->published_trailing_metadata && - stream_parsing->got_metadata_on_parse[1]) { - stream_parsing->got_metadata_on_parse[1] = 0; - stream_global->published_trailing_metadata = 1; - GPR_SWAP(grpc_chttp2_incoming_metadata_buffer, - stream_parsing->metadata_buffer[1], - stream_global->received_trailing_metadata); - grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global, - stream_global); - } - - if (stream_parsing->forced_close_error != GRPC_ERROR_NONE) { - intptr_t reason; - bool has_reason = grpc_error_get_int(stream_parsing->forced_close_error, - GRPC_ERROR_INT_HTTP2_ERROR, &reason); - if (has_reason && reason != GRPC_CHTTP2_NO_ERROR) { - grpc_status_code status_code = - has_reason - ? grpc_chttp2_http2_error_to_grpc_status( - (grpc_chttp2_error_code)reason, stream_global->deadline) - : GRPC_STATUS_INTERNAL; - const char *status_details = - grpc_error_string(stream_parsing->forced_close_error); - gpr_slice slice_details = gpr_slice_from_copied_string(status_details); - grpc_error_free_string(status_details); - grpc_chttp2_fake_status(exec_ctx, transport_global, stream_global, - status_code, &slice_details); - } - grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global, - 1, 1, stream_parsing->forced_close_error); - } - - if (stream_parsing->received_close) { - grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global, - 1, 0, GRPC_ERROR_NONE); - } - } -} - -grpc_error *grpc_chttp2_perform_read( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, - gpr_slice slice) { - uint8_t *beg = GPR_SLICE_START_PTR(slice); - uint8_t *end = GPR_SLICE_END_PTR(slice); +static grpc_error *init_frame_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); +static grpc_error *init_header_frame_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + int is_continuation); +static grpc_error *init_data_frame_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); +static grpc_error *init_rst_stream_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); +static grpc_error *init_settings_frame_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); +static grpc_error *init_window_update_frame_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); +static grpc_error *init_ping_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); +static grpc_error *init_goaway_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); +static grpc_error *init_skip_frame_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + int is_header); + +static grpc_error *parse_frame_slice(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, grpc_slice slice, + int is_last); + +grpc_error *grpc_chttp2_perform_read(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_slice slice) { + uint8_t *beg = GRPC_SLICE_START_PTR(slice); + uint8_t *end = GRPC_SLICE_END_PTR(slice); uint8_t *cur = beg; grpc_error *err; if (cur == end) return GRPC_ERROR_NONE; - switch (transport_parsing->deframe_state) { + switch (t->deframe_state) { case GRPC_DTS_CLIENT_PREFIX_0: case GRPC_DTS_CLIENT_PREFIX_1: case GRPC_DTS_CLIENT_PREFIX_2: @@ -298,25 +105,22 @@ grpc_error *grpc_chttp2_perform_read( case GRPC_DTS_CLIENT_PREFIX_21: case GRPC_DTS_CLIENT_PREFIX_22: case GRPC_DTS_CLIENT_PREFIX_23: - while (cur != end && transport_parsing->deframe_state != GRPC_DTS_FH_0) { - if (*cur != GRPC_CHTTP2_CLIENT_CONNECT_STRING[transport_parsing - ->deframe_state]) { + while (cur != end && t->deframe_state != GRPC_DTS_FH_0) { + if (*cur != GRPC_CHTTP2_CLIENT_CONNECT_STRING[t->deframe_state]) { char *msg; gpr_asprintf( &msg, "Connect string mismatch: expected '%c' (%d) got '%c' (%d) " "at byte %d", - GRPC_CHTTP2_CLIENT_CONNECT_STRING[transport_parsing - ->deframe_state], - (int)(uint8_t)GRPC_CHTTP2_CLIENT_CONNECT_STRING - [transport_parsing->deframe_state], - *cur, (int)*cur, transport_parsing->deframe_state); + GRPC_CHTTP2_CLIENT_CONNECT_STRING[t->deframe_state], + (int)(uint8_t)GRPC_CHTTP2_CLIENT_CONNECT_STRING[t->deframe_state], + *cur, (int)*cur, t->deframe_state); err = GRPC_ERROR_CREATE(msg); gpr_free(msg); return err; } ++cur; - ++transport_parsing->deframe_state; + ++t->deframe_state; } if (cur == end) { return GRPC_ERROR_NONE; @@ -325,100 +129,95 @@ grpc_error *grpc_chttp2_perform_read( dts_fh_0: case GRPC_DTS_FH_0: GPR_ASSERT(cur < end); - transport_parsing->incoming_frame_size = ((uint32_t)*cur) << 16; + t->incoming_frame_size = ((uint32_t)*cur) << 16; if (++cur == end) { - transport_parsing->deframe_state = GRPC_DTS_FH_1; + t->deframe_state = GRPC_DTS_FH_1; return GRPC_ERROR_NONE; } /* fallthrough */ case GRPC_DTS_FH_1: GPR_ASSERT(cur < end); - transport_parsing->incoming_frame_size |= ((uint32_t)*cur) << 8; + t->incoming_frame_size |= ((uint32_t)*cur) << 8; if (++cur == end) { - transport_parsing->deframe_state = GRPC_DTS_FH_2; + t->deframe_state = GRPC_DTS_FH_2; return GRPC_ERROR_NONE; } /* fallthrough */ case GRPC_DTS_FH_2: GPR_ASSERT(cur < end); - transport_parsing->incoming_frame_size |= *cur; + t->incoming_frame_size |= *cur; if (++cur == end) { - transport_parsing->deframe_state = GRPC_DTS_FH_3; + t->deframe_state = GRPC_DTS_FH_3; return GRPC_ERROR_NONE; } /* fallthrough */ case GRPC_DTS_FH_3: GPR_ASSERT(cur < end); - transport_parsing->incoming_frame_type = *cur; + t->incoming_frame_type = *cur; if (++cur == end) { - transport_parsing->deframe_state = GRPC_DTS_FH_4; + t->deframe_state = GRPC_DTS_FH_4; return GRPC_ERROR_NONE; } /* fallthrough */ case GRPC_DTS_FH_4: GPR_ASSERT(cur < end); - transport_parsing->incoming_frame_flags = *cur; + t->incoming_frame_flags = *cur; if (++cur == end) { - transport_parsing->deframe_state = GRPC_DTS_FH_5; + t->deframe_state = GRPC_DTS_FH_5; return GRPC_ERROR_NONE; } /* fallthrough */ case GRPC_DTS_FH_5: GPR_ASSERT(cur < end); - transport_parsing->incoming_stream_id = (((uint32_t)*cur) & 0x7f) << 24; + t->incoming_stream_id = (((uint32_t)*cur) & 0x7f) << 24; if (++cur == end) { - transport_parsing->deframe_state = GRPC_DTS_FH_6; + t->deframe_state = GRPC_DTS_FH_6; return GRPC_ERROR_NONE; } /* fallthrough */ case GRPC_DTS_FH_6: GPR_ASSERT(cur < end); - transport_parsing->incoming_stream_id |= ((uint32_t)*cur) << 16; + t->incoming_stream_id |= ((uint32_t)*cur) << 16; if (++cur == end) { - transport_parsing->deframe_state = GRPC_DTS_FH_7; + t->deframe_state = GRPC_DTS_FH_7; return GRPC_ERROR_NONE; } /* fallthrough */ case GRPC_DTS_FH_7: GPR_ASSERT(cur < end); - transport_parsing->incoming_stream_id |= ((uint32_t)*cur) << 8; + t->incoming_stream_id |= ((uint32_t)*cur) << 8; if (++cur == end) { - transport_parsing->deframe_state = GRPC_DTS_FH_8; + t->deframe_state = GRPC_DTS_FH_8; return GRPC_ERROR_NONE; } /* fallthrough */ case GRPC_DTS_FH_8: GPR_ASSERT(cur < end); - transport_parsing->incoming_stream_id |= ((uint32_t)*cur); - transport_parsing->deframe_state = GRPC_DTS_FRAME; - err = init_frame_parser(exec_ctx, transport_parsing); + t->incoming_stream_id |= ((uint32_t)*cur); + t->deframe_state = GRPC_DTS_FRAME; + err = init_frame_parser(exec_ctx, t); if (err != GRPC_ERROR_NONE) { return err; } - if (transport_parsing->incoming_stream_id != 0 && - transport_parsing->incoming_stream_id > - transport_parsing->last_incoming_stream_id) { - transport_parsing->last_incoming_stream_id = - transport_parsing->incoming_stream_id; - } - if (transport_parsing->incoming_frame_size == 0) { - err = parse_frame_slice(exec_ctx, transport_parsing, gpr_empty_slice(), - 1); + if (t->incoming_frame_size == 0) { + err = parse_frame_slice(exec_ctx, t, gpr_empty_slice(), 1); if (err != GRPC_ERROR_NONE) { return err; } - transport_parsing->incoming_stream = NULL; + t->incoming_stream = NULL; if (++cur == end) { - transport_parsing->deframe_state = GRPC_DTS_FH_0; + t->deframe_state = GRPC_DTS_FH_0; return GRPC_ERROR_NONE; } goto dts_fh_0; /* loop */ - } else if (transport_parsing->incoming_frame_size > - transport_parsing->max_frame_size) { + } else if (t->incoming_frame_size > + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]) { char *msg; gpr_asprintf(&msg, "Frame size %d is larger than max frame size %d", - transport_parsing->incoming_frame_size, - transport_parsing->max_frame_size); + t->incoming_frame_size, + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]); err = GRPC_ERROR_CREATE(msg); gpr_free(msg); return err; @@ -429,41 +228,39 @@ grpc_error *grpc_chttp2_perform_read( /* fallthrough */ case GRPC_DTS_FRAME: GPR_ASSERT(cur < end); - if ((uint32_t)(end - cur) == transport_parsing->incoming_frame_size) { - err = parse_frame_slice(exec_ctx, transport_parsing, - gpr_slice_sub_no_ref(slice, (size_t)(cur - beg), - (size_t)(end - beg)), - 1); + if ((uint32_t)(end - cur) == t->incoming_frame_size) { + err = parse_frame_slice( + exec_ctx, t, grpc_slice_sub_no_ref(slice, (size_t)(cur - beg), + (size_t)(end - beg)), + 1); if (err != GRPC_ERROR_NONE) { return err; } - transport_parsing->deframe_state = GRPC_DTS_FH_0; - transport_parsing->incoming_stream = NULL; + t->deframe_state = GRPC_DTS_FH_0; + t->incoming_stream = NULL; return GRPC_ERROR_NONE; - } else if ((uint32_t)(end - cur) > - transport_parsing->incoming_frame_size) { + } else if ((uint32_t)(end - cur) > t->incoming_frame_size) { size_t cur_offset = (size_t)(cur - beg); err = parse_frame_slice( - exec_ctx, transport_parsing, - gpr_slice_sub_no_ref( - slice, cur_offset, - cur_offset + transport_parsing->incoming_frame_size), + exec_ctx, t, + grpc_slice_sub_no_ref(slice, cur_offset, + cur_offset + t->incoming_frame_size), 1); if (err != GRPC_ERROR_NONE) { return err; } - cur += transport_parsing->incoming_frame_size; - transport_parsing->incoming_stream = NULL; + cur += t->incoming_frame_size; + t->incoming_stream = NULL; goto dts_fh_0; /* loop */ } else { - err = parse_frame_slice(exec_ctx, transport_parsing, - gpr_slice_sub_no_ref(slice, (size_t)(cur - beg), - (size_t)(end - beg)), - 0); + err = parse_frame_slice( + exec_ctx, t, grpc_slice_sub_no_ref(slice, (size_t)(cur - beg), + (size_t)(end - beg)), + 0); if (err != GRPC_ERROR_NONE) { return err; } - transport_parsing->incoming_frame_size -= (uint32_t)(end - cur); + t->incoming_frame_size -= (uint32_t)(end - cur); return GRPC_ERROR_NONE; } GPR_UNREACHABLE_CODE(return 0); @@ -472,175 +269,172 @@ grpc_error *grpc_chttp2_perform_read( GPR_UNREACHABLE_CODE(return 0); } -static grpc_error *init_frame_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { - if (transport_parsing->is_first_frame && - transport_parsing->incoming_frame_type != GRPC_CHTTP2_FRAME_SETTINGS) { +static grpc_error *init_frame_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { + if (t->is_first_frame && + t->incoming_frame_type != GRPC_CHTTP2_FRAME_SETTINGS) { char *msg; gpr_asprintf( &msg, "Expected SETTINGS frame as the first frame, got frame type %d", - transport_parsing->incoming_frame_type); + t->incoming_frame_type); grpc_error *err = GRPC_ERROR_CREATE(msg); gpr_free(msg); return err; } - transport_parsing->is_first_frame = false; - if (transport_parsing->expect_continuation_stream_id != 0) { - if (transport_parsing->incoming_frame_type != - GRPC_CHTTP2_FRAME_CONTINUATION) { + t->is_first_frame = false; + if (t->expect_continuation_stream_id != 0) { + if (t->incoming_frame_type != GRPC_CHTTP2_FRAME_CONTINUATION) { char *msg; gpr_asprintf(&msg, "Expected CONTINUATION frame, got frame type %02x", - transport_parsing->incoming_frame_type); + t->incoming_frame_type); grpc_error *err = GRPC_ERROR_CREATE(msg); gpr_free(msg); return err; } - if (transport_parsing->expect_continuation_stream_id != - transport_parsing->incoming_stream_id) { + if (t->expect_continuation_stream_id != t->incoming_stream_id) { char *msg; gpr_asprintf( &msg, "Expected CONTINUATION frame for grpc_chttp2_stream %08x, got " "grpc_chttp2_stream %08x", - transport_parsing->expect_continuation_stream_id, - transport_parsing->incoming_stream_id); + t->expect_continuation_stream_id, t->incoming_stream_id); grpc_error *err = GRPC_ERROR_CREATE(msg); gpr_free(msg); return err; } - return init_header_frame_parser(exec_ctx, transport_parsing, 1); + return init_header_frame_parser(exec_ctx, t, 1); } - switch (transport_parsing->incoming_frame_type) { + switch (t->incoming_frame_type) { case GRPC_CHTTP2_FRAME_DATA: - return init_data_frame_parser(exec_ctx, transport_parsing); + return init_data_frame_parser(exec_ctx, t); case GRPC_CHTTP2_FRAME_HEADER: - return init_header_frame_parser(exec_ctx, transport_parsing, 0); + return init_header_frame_parser(exec_ctx, t, 0); case GRPC_CHTTP2_FRAME_CONTINUATION: return GRPC_ERROR_CREATE("Unexpected CONTINUATION frame"); case GRPC_CHTTP2_FRAME_RST_STREAM: - return init_rst_stream_parser(exec_ctx, transport_parsing); + return init_rst_stream_parser(exec_ctx, t); case GRPC_CHTTP2_FRAME_SETTINGS: - return init_settings_frame_parser(exec_ctx, transport_parsing); + return init_settings_frame_parser(exec_ctx, t); case GRPC_CHTTP2_FRAME_WINDOW_UPDATE: - return init_window_update_frame_parser(exec_ctx, transport_parsing); + return init_window_update_frame_parser(exec_ctx, t); case GRPC_CHTTP2_FRAME_PING: - return init_ping_parser(exec_ctx, transport_parsing); + return init_ping_parser(exec_ctx, t); case GRPC_CHTTP2_FRAME_GOAWAY: - return init_goaway_parser(exec_ctx, transport_parsing); + return init_goaway_parser(exec_ctx, t); default: if (grpc_http_trace) { - gpr_log(GPR_ERROR, "Unknown frame type %02x", - transport_parsing->incoming_frame_type); + gpr_log(GPR_ERROR, "Unknown frame type %02x", t->incoming_frame_type); } - return init_skip_frame_parser(exec_ctx, transport_parsing, 0); + return init_skip_frame_parser(exec_ctx, t, 0); } } static grpc_error *skip_parser(grpc_exec_ctx *exec_ctx, void *parser, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing, - gpr_slice slice, int is_last) { + grpc_chttp2_transport *t, grpc_chttp2_stream *s, + grpc_slice slice, int is_last) { return GRPC_ERROR_NONE; } -static void skip_header(void *tp, grpc_mdelem *md) { GRPC_MDELEM_UNREF(md); } +static void skip_header(grpc_exec_ctx *exec_ctx, void *tp, grpc_mdelem *md) { + GRPC_MDELEM_UNREF(md); +} -static grpc_error *init_skip_frame_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, - int is_header) { +static grpc_error *init_skip_frame_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + int is_header) { if (is_header) { - uint8_t is_eoh = transport_parsing->expect_continuation_stream_id != 0; - transport_parsing->parser = grpc_chttp2_header_parser_parse; - transport_parsing->parser_data = &transport_parsing->hpack_parser; - transport_parsing->hpack_parser.on_header = skip_header; - transport_parsing->hpack_parser.on_header_user_data = NULL; - transport_parsing->hpack_parser.is_boundary = is_eoh; - transport_parsing->hpack_parser.is_eof = - (uint8_t)(is_eoh ? transport_parsing->header_eof : 0); + uint8_t is_eoh = t->expect_continuation_stream_id != 0; + t->parser = grpc_chttp2_header_parser_parse; + t->parser_data = &t->hpack_parser; + t->hpack_parser.on_header = skip_header; + t->hpack_parser.on_header_user_data = NULL; + t->hpack_parser.is_boundary = is_eoh; + t->hpack_parser.is_eof = (uint8_t)(is_eoh ? t->header_eof : 0); } else { - transport_parsing->parser = skip_parser; + t->parser = skip_parser; } return GRPC_ERROR_NONE; } -void grpc_chttp2_parsing_become_skip_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { - init_skip_frame_parser( - exec_ctx, transport_parsing, - transport_parsing->parser == grpc_chttp2_header_parser_parse); +void grpc_chttp2_parsing_become_skip_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { + init_skip_frame_parser(exec_ctx, t, + t->parser == grpc_chttp2_header_parser_parse); } -static grpc_error *update_incoming_window( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing) { - uint32_t incoming_frame_size = transport_parsing->incoming_frame_size; - if (incoming_frame_size > transport_parsing->incoming_window) { - char *msg; - gpr_asprintf(&msg, "frame of size %d overflows incoming window of %" PRId64, - transport_parsing->incoming_frame_size, - transport_parsing->incoming_window); - grpc_error *err = GRPC_ERROR_CREATE(msg); - gpr_free(msg); - return err; - } - - if (incoming_frame_size > stream_parsing->incoming_window) { +static grpc_error *update_incoming_window(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { + uint32_t incoming_frame_size = t->incoming_frame_size; + if (incoming_frame_size > t->incoming_window) { char *msg; gpr_asprintf(&msg, "frame of size %d overflows incoming window of %" PRId64, - transport_parsing->incoming_frame_size, - stream_parsing->incoming_window); + t->incoming_frame_size, t->incoming_window); grpc_error *err = GRPC_ERROR_CREATE(msg); gpr_free(msg); return err; } - GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("parse", transport_parsing, incoming_window, + GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("parse", t, incoming_window, incoming_frame_size); - GRPC_CHTTP2_FLOW_DEBIT_STREAM("parse", transport_parsing, stream_parsing, - incoming_window, incoming_frame_size); - stream_parsing->received_bytes += incoming_frame_size; - grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, stream_parsing); + if (s != NULL) { + if (incoming_frame_size > s->incoming_window) { + char *msg; + gpr_asprintf(&msg, + "frame of size %d overflows incoming window of %" PRId64, + t->incoming_frame_size, s->incoming_window); + grpc_error *err = GRPC_ERROR_CREATE(msg); + gpr_free(msg); + return err; + } + + GRPC_CHTTP2_FLOW_DEBIT_STREAM("parse", t, s, incoming_window, + incoming_frame_size); + s->received_bytes += incoming_frame_size; + s->max_recv_bytes -= + (uint32_t)GPR_MIN(s->max_recv_bytes, incoming_frame_size); + } return GRPC_ERROR_NONE; } -static grpc_error *init_data_frame_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { - grpc_chttp2_stream_parsing *stream_parsing = - grpc_chttp2_parsing_lookup_stream(transport_parsing, - transport_parsing->incoming_stream_id); +static grpc_error *init_data_frame_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { + grpc_chttp2_stream *s = + grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id); grpc_error *err = GRPC_ERROR_NONE; - if (stream_parsing == NULL) { - return init_skip_frame_parser(exec_ctx, transport_parsing, 0); + err = update_incoming_window(exec_ctx, t, s); + if (err != GRPC_ERROR_NONE) { + goto error_handler; } - stream_parsing->stats.incoming.framing_bytes += 9; - if (stream_parsing->received_close) { - return init_skip_frame_parser(exec_ctx, transport_parsing, 0); + if (s == NULL) { + return init_skip_frame_parser(exec_ctx, t, 0); } - if (err == GRPC_ERROR_NONE) { - err = update_incoming_window(exec_ctx, transport_parsing, stream_parsing); + s->stats.incoming.framing_bytes += 9; + if (err == GRPC_ERROR_NONE && s->read_closed) { + return init_skip_frame_parser(exec_ctx, t, 0); } if (err == GRPC_ERROR_NONE) { - err = grpc_chttp2_data_parser_begin_frame( - &stream_parsing->data_parser, transport_parsing->incoming_frame_flags, - stream_parsing->id); + err = grpc_chttp2_data_parser_begin_frame(&s->data_parser, + t->incoming_frame_flags, s->id); } +error_handler: if (err == GRPC_ERROR_NONE) { - transport_parsing->incoming_stream = stream_parsing; - transport_parsing->parser = grpc_chttp2_data_parser_parse; - transport_parsing->parser_data = &stream_parsing->data_parser; + t->incoming_stream = s; + t->parser = grpc_chttp2_data_parser_parse; + t->parser_data = &s->data_parser; return GRPC_ERROR_NONE; } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, NULL)) { /* handle stream errors by closing the stream */ - stream_parsing->received_close = 1; - stream_parsing->forced_close_error = err; - gpr_slice_buffer_add( - &transport_parsing->qbuf, - grpc_chttp2_rst_stream_create(transport_parsing->incoming_stream_id, - GRPC_CHTTP2_PROTOCOL_ERROR, - &stream_parsing->stats.outgoing)); - return init_skip_frame_parser(exec_ctx, transport_parsing, 0); + if (s != NULL) { + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, false, err); + } + grpc_slice_buffer_add( + &t->qbuf, grpc_chttp2_rst_stream_create(t->incoming_stream_id, + GRPC_CHTTP2_PROTOCOL_ERROR, + &s->stats.outgoing)); + return init_skip_frame_parser(exec_ctx, t, 0); } else { return err; } @@ -648,23 +442,22 @@ static grpc_error *init_data_frame_parser( static void free_timeout(void *p) { gpr_free(p); } -static void on_initial_header(void *tp, grpc_mdelem *md) { - grpc_chttp2_transport_parsing *transport_parsing = tp; - grpc_chttp2_stream_parsing *stream_parsing = - transport_parsing->incoming_stream; +static void on_initial_header(grpc_exec_ctx *exec_ctx, void *tp, + grpc_mdelem *md) { + grpc_chttp2_transport *t = tp; + grpc_chttp2_stream *s = t->incoming_stream; GPR_TIMER_BEGIN("on_initial_header", 0); - GPR_ASSERT(stream_parsing); + GPR_ASSERT(s != NULL); GRPC_CHTTP2_IF_TRACING(gpr_log( - GPR_INFO, "HTTP:%d:HDR:%s: %s: %s", stream_parsing->id, - transport_parsing->is_client ? "CLI" : "SVR", + GPR_INFO, "HTTP:%d:HDR:%s: %s: %s", s->id, t->is_client ? "CLI" : "SVR", grpc_mdstr_as_c_string(md->key), grpc_mdstr_as_c_string(md->value))); if (md->key == GRPC_MDSTR_GRPC_STATUS && md != GRPC_MDELEM_GRPC_STATUS_0) { /* TODO(ctiller): check for a status like " 0" */ - stream_parsing->seen_error = true; + s->seen_error = true; } if (md->key == GRPC_MDSTR_GRPC_TIMEOUT) { @@ -681,306 +474,273 @@ static void on_initial_header(void *tp, grpc_mdelem *md) { grpc_mdelem_set_user_data(md, free_timeout, cached_timeout); } grpc_chttp2_incoming_metadata_buffer_set_deadline( - &stream_parsing->metadata_buffer[0], + &s->metadata_buffer[0], gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), *cached_timeout)); GRPC_MDELEM_UNREF(md); } else { - const size_t new_size = - stream_parsing->metadata_buffer[0].size + GRPC_MDELEM_LENGTH(md); - grpc_chttp2_transport_global *transport_global = - &TRANSPORT_FROM_PARSING(transport_parsing)->global; + const size_t new_size = s->metadata_buffer[0].size + GRPC_MDELEM_LENGTH(md); const size_t metadata_size_limit = - transport_global->settings[GRPC_LOCAL_SETTINGS] - [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; if (new_size > metadata_size_limit) { - if (!stream_parsing->exceeded_metadata_size) { - gpr_log(GPR_DEBUG, - "received initial metadata size exceeds limit (%" PRIuPTR - " vs. %" PRIuPTR ")", - new_size, metadata_size_limit); - stream_parsing->seen_error = true; - stream_parsing->exceeded_metadata_size = true; - } + gpr_log(GPR_DEBUG, + "received initial metadata size exceeds limit (%" PRIuPTR + " vs. %" PRIuPTR ")", + new_size, metadata_size_limit); + grpc_chttp2_cancel_stream( + exec_ctx, t, s, + grpc_error_set_int( + GRPC_ERROR_CREATE("received initial metadata size exceeds limit"), + GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED)); + grpc_chttp2_parsing_become_skip_parser(exec_ctx, t); + s->seen_error = true; GRPC_MDELEM_UNREF(md); } else { - grpc_chttp2_incoming_metadata_buffer_add( - &stream_parsing->metadata_buffer[0], md); + grpc_chttp2_incoming_metadata_buffer_add(&s->metadata_buffer[0], md); } } - grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, stream_parsing); - GPR_TIMER_END("on_initial_header", 0); } -static void on_trailing_header(void *tp, grpc_mdelem *md) { - grpc_chttp2_transport_parsing *transport_parsing = tp; - grpc_chttp2_stream_parsing *stream_parsing = - transport_parsing->incoming_stream; +static void on_trailing_header(grpc_exec_ctx *exec_ctx, void *tp, + grpc_mdelem *md) { + grpc_chttp2_transport *t = tp; + grpc_chttp2_stream *s = t->incoming_stream; GPR_TIMER_BEGIN("on_trailing_header", 0); - GPR_ASSERT(stream_parsing); + GPR_ASSERT(s != NULL); GRPC_CHTTP2_IF_TRACING(gpr_log( - GPR_INFO, "HTTP:%d:TRL:%s: %s: %s", stream_parsing->id, - transport_parsing->is_client ? "CLI" : "SVR", + GPR_INFO, "HTTP:%d:TRL:%s: %s: %s", s->id, t->is_client ? "CLI" : "SVR", grpc_mdstr_as_c_string(md->key), grpc_mdstr_as_c_string(md->value))); if (md->key == GRPC_MDSTR_GRPC_STATUS && md != GRPC_MDELEM_GRPC_STATUS_0) { /* TODO(ctiller): check for a status like " 0" */ - stream_parsing->seen_error = true; + s->seen_error = true; } - const size_t new_size = - stream_parsing->metadata_buffer[1].size + GRPC_MDELEM_LENGTH(md); - grpc_chttp2_transport_global *transport_global = - &TRANSPORT_FROM_PARSING(transport_parsing)->global; + const size_t new_size = s->metadata_buffer[1].size + GRPC_MDELEM_LENGTH(md); const size_t metadata_size_limit = - transport_global->settings[GRPC_LOCAL_SETTINGS] - [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE]; if (new_size > metadata_size_limit) { - if (!stream_parsing->exceeded_metadata_size) { - gpr_log(GPR_DEBUG, - "received trailing metadata size exceeds limit (%" PRIuPTR - " vs. %" PRIuPTR ")", - new_size, metadata_size_limit); - stream_parsing->seen_error = true; - stream_parsing->exceeded_metadata_size = true; - } + gpr_log(GPR_DEBUG, + "received trailing metadata size exceeds limit (%" PRIuPTR + " vs. %" PRIuPTR ")", + new_size, metadata_size_limit); + grpc_chttp2_cancel_stream( + exec_ctx, t, s, + grpc_error_set_int( + GRPC_ERROR_CREATE("received trailing metadata size exceeds limit"), + GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_RESOURCE_EXHAUSTED)); + grpc_chttp2_parsing_become_skip_parser(exec_ctx, t); + s->seen_error = true; GRPC_MDELEM_UNREF(md); } else { - grpc_chttp2_incoming_metadata_buffer_add( - &stream_parsing->metadata_buffer[1], md); + grpc_chttp2_incoming_metadata_buffer_add(&s->metadata_buffer[1], md); } - grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, stream_parsing); - GPR_TIMER_END("on_trailing_header", 0); } -static grpc_error *init_header_frame_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, - int is_continuation) { - uint8_t is_eoh = (transport_parsing->incoming_frame_flags & - GRPC_CHTTP2_DATA_FLAG_END_HEADERS) != 0; - int via_accept = 0; - grpc_chttp2_stream_parsing *stream_parsing; +static grpc_error *init_header_frame_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, + int is_continuation) { + uint8_t is_eoh = + (t->incoming_frame_flags & GRPC_CHTTP2_DATA_FLAG_END_HEADERS) != 0; + grpc_chttp2_stream *s; /* TODO(ctiller): when to increment header_frames_received? */ if (is_eoh) { - transport_parsing->expect_continuation_stream_id = 0; + t->expect_continuation_stream_id = 0; } else { - transport_parsing->expect_continuation_stream_id = - transport_parsing->incoming_stream_id; + t->expect_continuation_stream_id = t->incoming_stream_id; } if (!is_continuation) { - transport_parsing->header_eof = (transport_parsing->incoming_frame_flags & - GRPC_CHTTP2_DATA_FLAG_END_STREAM) != 0; + t->header_eof = + (t->incoming_frame_flags & GRPC_CHTTP2_DATA_FLAG_END_STREAM) != 0; } /* could be a new grpc_chttp2_stream or an existing grpc_chttp2_stream */ - stream_parsing = grpc_chttp2_parsing_lookup_stream( - transport_parsing, transport_parsing->incoming_stream_id); - if (stream_parsing == NULL) { + s = grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id); + if (s == NULL) { if (is_continuation) { - gpr_log(GPR_ERROR, - "grpc_chttp2_stream disbanded before CONTINUATION received"); - return init_skip_frame_parser(exec_ctx, transport_parsing, 1); + GRPC_CHTTP2_IF_TRACING( + gpr_log(GPR_ERROR, + "grpc_chttp2_stream disbanded before CONTINUATION received")); + return init_skip_frame_parser(exec_ctx, t, 1); } - if (transport_parsing->is_client) { - if ((transport_parsing->incoming_stream_id & 1) && - transport_parsing->incoming_stream_id < - transport_parsing->next_stream_id) { + if (t->is_client) { + if ((t->incoming_stream_id & 1) && + t->incoming_stream_id < t->next_stream_id) { /* this is an old (probably cancelled) grpc_chttp2_stream */ } else { - gpr_log(GPR_ERROR, - "ignoring new grpc_chttp2_stream creation on client"); + GRPC_CHTTP2_IF_TRACING(gpr_log( + GPR_ERROR, "ignoring new grpc_chttp2_stream creation on client")); } - return init_skip_frame_parser(exec_ctx, transport_parsing, 1); - } else if (transport_parsing->last_incoming_stream_id > - transport_parsing->incoming_stream_id) { - gpr_log(GPR_ERROR, - "ignoring out of order new grpc_chttp2_stream request on server; " - "last grpc_chttp2_stream " - "id=%d, new grpc_chttp2_stream id=%d", - transport_parsing->last_incoming_stream_id, - transport_parsing->incoming_stream_id); - return init_skip_frame_parser(exec_ctx, transport_parsing, 1); - } else if ((transport_parsing->incoming_stream_id & 1) == 0) { - gpr_log(GPR_ERROR, - "ignoring grpc_chttp2_stream with non-client generated index %d", - transport_parsing->incoming_stream_id); - return init_skip_frame_parser(exec_ctx, transport_parsing, 1); + return init_skip_frame_parser(exec_ctx, t, 1); + } else if (t->last_new_stream_id >= t->incoming_stream_id) { + GRPC_CHTTP2_IF_TRACING(gpr_log( + GPR_ERROR, + "ignoring out of order new grpc_chttp2_stream request on server; " + "last grpc_chttp2_stream " + "id=%d, new grpc_chttp2_stream id=%d", + t->last_new_stream_id, t->incoming_stream_id)); + return init_skip_frame_parser(exec_ctx, t, 1); + } else if ((t->incoming_stream_id & 1) == 0) { + GRPC_CHTTP2_IF_TRACING(gpr_log( + GPR_ERROR, + "ignoring grpc_chttp2_stream with non-client generated index %d", + t->incoming_stream_id)); + return init_skip_frame_parser(exec_ctx, t, 1); } - stream_parsing = transport_parsing->incoming_stream = - grpc_chttp2_parsing_accept_stream( - exec_ctx, transport_parsing, transport_parsing->incoming_stream_id); - if (stream_parsing == NULL) { - gpr_log(GPR_ERROR, "grpc_chttp2_stream not accepted"); - return init_skip_frame_parser(exec_ctx, transport_parsing, 1); + t->last_new_stream_id = t->incoming_stream_id; + s = t->incoming_stream = + grpc_chttp2_parsing_accept_stream(exec_ctx, t, t->incoming_stream_id); + if (s == NULL) { + GRPC_CHTTP2_IF_TRACING( + gpr_log(GPR_ERROR, "grpc_chttp2_stream not accepted")); + return init_skip_frame_parser(exec_ctx, t, 1); } - via_accept = 1; } else { - transport_parsing->incoming_stream = stream_parsing; - } - GPR_ASSERT(stream_parsing != NULL && (via_accept == 0 || via_accept == 1)); - stream_parsing->stats.incoming.framing_bytes += 9; - if (stream_parsing->received_close) { - gpr_log(GPR_ERROR, "skipping already closed grpc_chttp2_stream header"); - transport_parsing->incoming_stream = NULL; - return init_skip_frame_parser(exec_ctx, transport_parsing, 1); - } - transport_parsing->parser = grpc_chttp2_header_parser_parse; - transport_parsing->parser_data = &transport_parsing->hpack_parser; - switch (stream_parsing->header_frames_received) { + t->incoming_stream = s; + } + GPR_ASSERT(s != NULL); + s->stats.incoming.framing_bytes += 9; + if (s->read_closed) { + GRPC_CHTTP2_IF_TRACING(gpr_log( + GPR_ERROR, "skipping already closed grpc_chttp2_stream header")); + t->incoming_stream = NULL; + return init_skip_frame_parser(exec_ctx, t, 1); + } + t->parser = grpc_chttp2_header_parser_parse; + t->parser_data = &t->hpack_parser; + switch (s->header_frames_received) { case 0: - transport_parsing->hpack_parser.on_header = on_initial_header; + t->hpack_parser.on_header = on_initial_header; break; case 1: - transport_parsing->hpack_parser.on_header = on_trailing_header; + t->hpack_parser.on_header = on_trailing_header; break; case 2: gpr_log(GPR_ERROR, "too many header frames received"); - return init_skip_frame_parser(exec_ctx, transport_parsing, 1); + return init_skip_frame_parser(exec_ctx, t, 1); } - transport_parsing->hpack_parser.on_header_user_data = transport_parsing; - transport_parsing->hpack_parser.is_boundary = is_eoh; - transport_parsing->hpack_parser.is_eof = - (uint8_t)(is_eoh ? transport_parsing->header_eof : 0); - if (!is_continuation && (transport_parsing->incoming_frame_flags & - GRPC_CHTTP2_FLAG_HAS_PRIORITY)) { - grpc_chttp2_hpack_parser_set_has_priority(&transport_parsing->hpack_parser); + t->hpack_parser.on_header_user_data = t; + t->hpack_parser.is_boundary = is_eoh; + t->hpack_parser.is_eof = (uint8_t)(is_eoh ? t->header_eof : 0); + if (!is_continuation && + (t->incoming_frame_flags & GRPC_CHTTP2_FLAG_HAS_PRIORITY)) { + grpc_chttp2_hpack_parser_set_has_priority(&t->hpack_parser); } return GRPC_ERROR_NONE; } -static grpc_error *init_window_update_frame_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { +static grpc_error *init_window_update_frame_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { grpc_error *err = grpc_chttp2_window_update_parser_begin_frame( - &transport_parsing->simple.window_update, - transport_parsing->incoming_frame_size, - transport_parsing->incoming_frame_flags); + &t->simple.window_update, t->incoming_frame_size, + t->incoming_frame_flags); if (err != GRPC_ERROR_NONE) return err; - if (transport_parsing->incoming_stream_id != 0) { - grpc_chttp2_stream_parsing *stream_parsing = - transport_parsing->incoming_stream = grpc_chttp2_parsing_lookup_stream( - transport_parsing, transport_parsing->incoming_stream_id); - if (stream_parsing == NULL) { - return init_skip_frame_parser(exec_ctx, transport_parsing, 0); + if (t->incoming_stream_id != 0) { + grpc_chttp2_stream *s = t->incoming_stream = + grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id); + if (s == NULL) { + return init_skip_frame_parser(exec_ctx, t, 0); } - stream_parsing->stats.incoming.framing_bytes += 9; + s->stats.incoming.framing_bytes += 9; } - transport_parsing->parser = grpc_chttp2_window_update_parser_parse; - transport_parsing->parser_data = &transport_parsing->simple.window_update; + t->parser = grpc_chttp2_window_update_parser_parse; + t->parser_data = &t->simple.window_update; return GRPC_ERROR_NONE; } -static grpc_error *init_ping_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { +static grpc_error *init_ping_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { grpc_error *err = grpc_chttp2_ping_parser_begin_frame( - &transport_parsing->simple.ping, transport_parsing->incoming_frame_size, - transport_parsing->incoming_frame_flags); + &t->simple.ping, t->incoming_frame_size, t->incoming_frame_flags); if (err != GRPC_ERROR_NONE) return err; - transport_parsing->parser = grpc_chttp2_ping_parser_parse; - transport_parsing->parser_data = &transport_parsing->simple.ping; + t->parser = grpc_chttp2_ping_parser_parse; + t->parser_data = &t->simple.ping; return GRPC_ERROR_NONE; } -static grpc_error *init_rst_stream_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { +static grpc_error *init_rst_stream_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { grpc_error *err = grpc_chttp2_rst_stream_parser_begin_frame( - &transport_parsing->simple.rst_stream, - transport_parsing->incoming_frame_size, - transport_parsing->incoming_frame_flags); + &t->simple.rst_stream, t->incoming_frame_size, t->incoming_frame_flags); if (err != GRPC_ERROR_NONE) return err; - grpc_chttp2_stream_parsing *stream_parsing = - transport_parsing->incoming_stream = grpc_chttp2_parsing_lookup_stream( - transport_parsing, transport_parsing->incoming_stream_id); - if (!transport_parsing->incoming_stream) { - return init_skip_frame_parser(exec_ctx, transport_parsing, 0); - } - stream_parsing->stats.incoming.framing_bytes += 9; - transport_parsing->parser = grpc_chttp2_rst_stream_parser_parse; - transport_parsing->parser_data = &transport_parsing->simple.rst_stream; + grpc_chttp2_stream *s = t->incoming_stream = + grpc_chttp2_parsing_lookup_stream(t, t->incoming_stream_id); + if (!t->incoming_stream) { + return init_skip_frame_parser(exec_ctx, t, 0); + } + s->stats.incoming.framing_bytes += 9; + t->parser = grpc_chttp2_rst_stream_parser_parse; + t->parser_data = &t->simple.rst_stream; return GRPC_ERROR_NONE; } -static grpc_error *init_goaway_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { +static grpc_error *init_goaway_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { grpc_error *err = grpc_chttp2_goaway_parser_begin_frame( - &transport_parsing->goaway_parser, transport_parsing->incoming_frame_size, - transport_parsing->incoming_frame_flags); + &t->goaway_parser, t->incoming_frame_size, t->incoming_frame_flags); if (err != GRPC_ERROR_NONE) return err; - transport_parsing->parser = grpc_chttp2_goaway_parser_parse; - transport_parsing->parser_data = &transport_parsing->goaway_parser; + t->parser = grpc_chttp2_goaway_parser_parse; + t->parser_data = &t->goaway_parser; return GRPC_ERROR_NONE; } -static grpc_error *init_settings_frame_parser( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing) { - if (transport_parsing->incoming_stream_id != 0) { +static grpc_error *init_settings_frame_parser(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { + if (t->incoming_stream_id != 0) { return GRPC_ERROR_CREATE("Settings frame received for grpc_chttp2_stream"); } grpc_error *err = grpc_chttp2_settings_parser_begin_frame( - &transport_parsing->simple.settings, - transport_parsing->incoming_frame_size, - transport_parsing->incoming_frame_flags, transport_parsing->settings); + &t->simple.settings, t->incoming_frame_size, t->incoming_frame_flags, + t->settings[GRPC_PEER_SETTINGS]); if (err != GRPC_ERROR_NONE) { return err; } - if (transport_parsing->incoming_frame_flags & GRPC_CHTTP2_FLAG_ACK) { - transport_parsing->settings_ack_received = 1; + if (t->incoming_frame_flags & GRPC_CHTTP2_FLAG_ACK) { + memcpy(t->settings[GRPC_ACKED_SETTINGS], t->settings[GRPC_SENT_SETTINGS], + GRPC_CHTTP2_NUM_SETTINGS * sizeof(uint32_t)); grpc_chttp2_hptbl_set_max_bytes( - &transport_parsing->hpack_parser.table, - transport_parsing - ->last_sent_settings[GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]); - transport_parsing->max_frame_size = - transport_parsing - ->last_sent_settings[GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]; + &t->hpack_parser.table, + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]); + t->sent_local_settings = 0; } - transport_parsing->parser = grpc_chttp2_settings_parser_parse; - transport_parsing->parser_data = &transport_parsing->simple.settings; + t->parser = grpc_chttp2_settings_parser_parse; + t->parser_data = &t->simple.settings; return GRPC_ERROR_NONE; } -/* -static int is_window_update_legal(int64_t window_update, int64_t window) { - return window + window_update < MAX_WINDOW; -} -*/ - -static grpc_error *parse_frame_slice( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_parsing *transport_parsing, - gpr_slice slice, int is_last) { - grpc_chttp2_stream_parsing *stream_parsing = - transport_parsing->incoming_stream; - grpc_error *err = transport_parsing->parser( - exec_ctx, transport_parsing->parser_data, transport_parsing, - stream_parsing, slice, is_last); +static grpc_error *parse_frame_slice(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t, grpc_slice slice, + int is_last) { + grpc_chttp2_stream *s = t->incoming_stream; + grpc_error *err = t->parser(exec_ctx, t->parser_data, t, s, slice, is_last); if (err == GRPC_ERROR_NONE) { - if (stream_parsing) { - grpc_chttp2_list_add_parsing_seen_stream(transport_parsing, - stream_parsing); - } - return GRPC_ERROR_NONE; + return err; } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, NULL)) { if (grpc_http_trace) { const char *msg = grpc_error_string(err); gpr_log(GPR_ERROR, "%s", msg); grpc_error_free_string(msg); } - grpc_chttp2_parsing_become_skip_parser(exec_ctx, transport_parsing); - if (stream_parsing) { - stream_parsing->forced_close_error = err; - gpr_slice_buffer_add( - &transport_parsing->qbuf, - grpc_chttp2_rst_stream_create(transport_parsing->incoming_stream_id, - GRPC_CHTTP2_PROTOCOL_ERROR, - &stream_parsing->stats.outgoing)); + grpc_chttp2_parsing_become_skip_parser(exec_ctx, t); + if (s) { + s->forced_close_error = err; + grpc_slice_buffer_add( + &t->qbuf, grpc_chttp2_rst_stream_create(t->incoming_stream_id, + GRPC_CHTTP2_PROTOCOL_ERROR, + &s->stats.outgoing)); } else { GRPC_ERROR_UNREF(err); } diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.c b/src/core/ext/transport/chttp2/transport/stream_lists.c index 4dc4968248..6d25b3ae57 100644 --- a/src/core/ext/transport/chttp2/transport/stream_lists.c +++ b/src/core/ext/transport/chttp2/transport/stream_lists.c @@ -35,27 +35,6 @@ #include <grpc/support/log.h> -#define TRANSPORT_FROM_GLOBAL(tg) \ - ((grpc_chttp2_transport *)((char *)(tg)-offsetof(grpc_chttp2_transport, \ - global))) - -#define STREAM_FROM_GLOBAL(sg) \ - ((grpc_chttp2_stream *)((char *)(sg)-offsetof(grpc_chttp2_stream, global))) - -#define TRANSPORT_FROM_WRITING(tw) \ - ((grpc_chttp2_transport *)((char *)(tw)-offsetof(grpc_chttp2_transport, \ - writing))) - -#define STREAM_FROM_WRITING(sw) \ - ((grpc_chttp2_stream *)((char *)(sw)-offsetof(grpc_chttp2_stream, writing))) - -#define TRANSPORT_FROM_PARSING(tp) \ - ((grpc_chttp2_transport *)((char *)(tp)-offsetof(grpc_chttp2_transport, \ - parsing))) - -#define STREAM_FROM_PARSING(sp) \ - ((grpc_chttp2_stream *)((char *)(sp)-offsetof(grpc_chttp2_stream, parsing))) - /* core list management */ static int stream_list_empty(grpc_chttp2_transport *t, @@ -139,321 +118,57 @@ static bool stream_list_add(grpc_chttp2_transport *t, grpc_chttp2_stream *s, /* wrappers for specializations */ -bool grpc_chttp2_list_add_writable_stream( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) { - GPR_ASSERT(stream_global->id != 0); - return stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_WRITABLE); -} - -int grpc_chttp2_list_pop_writable_stream( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_global **stream_global, - grpc_chttp2_stream_writing **stream_writing) { - grpc_chttp2_stream *stream; - int r = stream_list_pop(TRANSPORT_FROM_GLOBAL(transport_global), &stream, - GRPC_CHTTP2_LIST_WRITABLE); - if (r != 0) { - *stream_global = &stream->global; - *stream_writing = &stream->writing; - } - return r; -} - -bool grpc_chttp2_list_remove_writable_stream( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) { - return stream_list_maybe_remove(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_WRITABLE); -} - -void grpc_chttp2_list_add_writing_stream( - grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_writing *stream_writing) { - GPR_ASSERT(stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), - STREAM_FROM_WRITING(stream_writing), - GRPC_CHTTP2_LIST_WRITING)); -} - -int grpc_chttp2_list_have_writing_streams( - grpc_chttp2_transport_writing *transport_writing) { - return !stream_list_empty(TRANSPORT_FROM_WRITING(transport_writing), - GRPC_CHTTP2_LIST_WRITING); -} - -int grpc_chttp2_list_pop_writing_stream( - grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_writing **stream_writing) { - grpc_chttp2_stream *stream; - int r = stream_list_pop(TRANSPORT_FROM_WRITING(transport_writing), &stream, - GRPC_CHTTP2_LIST_WRITING); - if (r != 0) { - *stream_writing = &stream->writing; - } - return r; -} - -void grpc_chttp2_list_add_written_stream( - grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_writing *stream_writing) { - stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), - STREAM_FROM_WRITING(stream_writing), - GRPC_CHTTP2_LIST_WRITTEN); -} - -int grpc_chttp2_list_pop_written_stream( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_global **stream_global, - grpc_chttp2_stream_writing **stream_writing) { - grpc_chttp2_stream *stream; - int r = stream_list_pop(TRANSPORT_FROM_WRITING(transport_writing), &stream, - GRPC_CHTTP2_LIST_WRITTEN); - if (r != 0) { - *stream_global = &stream->global; - *stream_writing = &stream->writing; - } - return r; -} - -void grpc_chttp2_list_add_unannounced_incoming_window_available( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) { - GPR_ASSERT(stream_global->id != 0); - stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_UNANNOUNCED_INCOMING_WINDOW_AVAILABLE); -} - -void grpc_chttp2_list_remove_unannounced_incoming_window_available( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) { - stream_list_maybe_remove( - TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_UNANNOUNCED_INCOMING_WINDOW_AVAILABLE); -} - -int grpc_chttp2_list_pop_unannounced_incoming_window_available( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_global **stream_global, - grpc_chttp2_stream_parsing **stream_parsing) { - grpc_chttp2_stream *stream; - int r = - stream_list_pop(TRANSPORT_FROM_GLOBAL(transport_global), &stream, - GRPC_CHTTP2_LIST_UNANNOUNCED_INCOMING_WINDOW_AVAILABLE); - if (r != 0) { - *stream_global = &stream->global; - *stream_parsing = &stream->parsing; - } - return r; -} - -void grpc_chttp2_list_add_parsing_seen_stream( - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_parsing *stream_parsing) { - stream_list_add(TRANSPORT_FROM_PARSING(transport_parsing), - STREAM_FROM_PARSING(stream_parsing), - GRPC_CHTTP2_LIST_PARSING_SEEN); -} - -int grpc_chttp2_list_pop_parsing_seen_stream( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_parsing *transport_parsing, - grpc_chttp2_stream_global **stream_global, - grpc_chttp2_stream_parsing **stream_parsing) { - grpc_chttp2_stream *stream; - int r = stream_list_pop(TRANSPORT_FROM_PARSING(transport_parsing), &stream, - GRPC_CHTTP2_LIST_PARSING_SEEN); - if (r != 0) { - *stream_global = &stream->global; - *stream_parsing = &stream->parsing; - } - return r; -} - -void grpc_chttp2_list_add_waiting_for_concurrency( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) { - stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY); -} - -int grpc_chttp2_list_pop_waiting_for_concurrency( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global **stream_global) { - grpc_chttp2_stream *stream; - int r = stream_list_pop(TRANSPORT_FROM_GLOBAL(transport_global), &stream, - GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY); - if (r != 0) { - *stream_global = &stream->global; - } - return r; -} - -void grpc_chttp2_list_add_check_read_ops( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) { - grpc_chttp2_transport *t = TRANSPORT_FROM_GLOBAL(transport_global); - if (!t->executor.check_read_ops_scheduled) { - grpc_combiner_execute_finally(exec_ctx, t->executor.combiner, - &t->initiate_read_flush_locked, - GRPC_ERROR_NONE, false); - t->executor.check_read_ops_scheduled = true; - } - stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_CHECK_READ_OPS); -} - -bool grpc_chttp2_list_remove_check_read_ops( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) { - return stream_list_maybe_remove(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_CHECK_READ_OPS); +bool grpc_chttp2_list_add_writable_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { + GPR_ASSERT(s->id != 0); + return stream_list_add(t, s, GRPC_CHTTP2_LIST_WRITABLE); } -int grpc_chttp2_list_pop_check_read_ops( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global **stream_global) { - grpc_chttp2_stream *stream; - int r = stream_list_pop(TRANSPORT_FROM_GLOBAL(transport_global), &stream, - GRPC_CHTTP2_LIST_CHECK_READ_OPS); - if (r != 0) { - *stream_global = &stream->global; - } - return r; -} - -void grpc_chttp2_list_add_writing_stalled_by_transport( - grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_writing *stream_writing) { - grpc_chttp2_stream *stream = STREAM_FROM_WRITING(stream_writing); - gpr_log(GPR_DEBUG, "writing stalled %d", stream->global.id); - if (!stream->included[GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT]) { - GRPC_CHTTP2_STREAM_REF(&stream->global, "chttp2_writing_stalled"); - } - stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), stream, - GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT); +int grpc_chttp2_list_pop_writable_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream **s) { + return stream_list_pop(t, s, GRPC_CHTTP2_LIST_WRITABLE); } -bool grpc_chttp2_list_flush_writing_stalled_by_transport( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing) { - grpc_chttp2_stream *stream; - bool out = false; - grpc_chttp2_transport *transport = TRANSPORT_FROM_WRITING(transport_writing); - while (stream_list_pop(transport, &stream, - GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT)) { - gpr_log(GPR_DEBUG, "move %d from writing stalled to just stalled", - stream->global.id); - grpc_chttp2_list_add_stalled_by_transport(transport_writing, - &stream->writing); - GRPC_CHTTP2_STREAM_UNREF(exec_ctx, &stream->global, - "chttp2_writing_stalled"); - out = true; - } - return out; +bool grpc_chttp2_list_remove_writable_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { + return stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_WRITABLE); } -void grpc_chttp2_list_add_stalled_by_transport( - grpc_chttp2_transport_writing *transport_writing, - grpc_chttp2_stream_writing *stream_writing) { - gpr_log(GPR_DEBUG, "stalled %d", stream_writing->id); - stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), - STREAM_FROM_WRITING(stream_writing), - GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); +bool grpc_chttp2_list_add_writing_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { + return stream_list_add(t, s, GRPC_CHTTP2_LIST_WRITING); } -int grpc_chttp2_list_pop_stalled_by_transport( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global **stream_global) { - grpc_chttp2_stream *stream; - int r = stream_list_pop(TRANSPORT_FROM_GLOBAL(transport_global), &stream, - GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); - if (r != 0) { - *stream_global = &stream->global; - } - return r; +int grpc_chttp2_list_have_writing_streams(grpc_chttp2_transport *t) { + return !stream_list_empty(t, GRPC_CHTTP2_LIST_WRITING); } -void grpc_chttp2_list_remove_stalled_by_transport( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) { - stream_list_maybe_remove(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); +int grpc_chttp2_list_pop_writing_stream(grpc_chttp2_transport *t, + grpc_chttp2_stream **s) { + return stream_list_pop(t, s, GRPC_CHTTP2_LIST_WRITING); } -void grpc_chttp2_list_add_closed_waiting_for_parsing( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) { - stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_PARSING); +void grpc_chttp2_list_add_waiting_for_concurrency(grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { + stream_list_add(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY); } -int grpc_chttp2_list_pop_closed_waiting_for_parsing( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global **stream_global) { - grpc_chttp2_stream *stream; - int r = stream_list_pop(TRANSPORT_FROM_GLOBAL(transport_global), &stream, - GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_PARSING); - if (r != 0) { - *stream_global = &stream->global; - } - return r; +int grpc_chttp2_list_pop_waiting_for_concurrency(grpc_chttp2_transport *t, + grpc_chttp2_stream **s) { + return stream_list_pop(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY); } -void grpc_chttp2_list_add_closed_waiting_for_writing( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global *stream_global) { - stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global), - STREAM_FROM_GLOBAL(stream_global), - GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_WRITING); +void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { + stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } -int grpc_chttp2_list_pop_closed_waiting_for_writing( - grpc_chttp2_transport_global *transport_global, - grpc_chttp2_stream_global **stream_global) { - grpc_chttp2_stream *stream; - int r = stream_list_pop(TRANSPORT_FROM_GLOBAL(transport_global), &stream, - GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_WRITING); - if (r != 0) { - *stream_global = &stream->global; - } - return r; +int grpc_chttp2_list_pop_stalled_by_transport(grpc_chttp2_transport *t, + grpc_chttp2_stream **s) { + return stream_list_pop(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } -void grpc_chttp2_register_stream(grpc_chttp2_transport *t, - grpc_chttp2_stream *s) { - stream_list_add_tail(t, s, GRPC_CHTTP2_LIST_ALL_STREAMS); -} - -int grpc_chttp2_unregister_stream(grpc_chttp2_transport *t, - grpc_chttp2_stream *s) { - stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_ALL_STREAMS); - return stream_list_empty(t, GRPC_CHTTP2_LIST_ALL_STREAMS); -} - -int grpc_chttp2_has_streams(grpc_chttp2_transport *t) { - return !stream_list_empty(t, GRPC_CHTTP2_LIST_ALL_STREAMS); -} - -void grpc_chttp2_for_all_streams( - grpc_chttp2_transport_global *transport_global, void *user_data, - void (*cb)(grpc_chttp2_transport_global *transport_global, void *user_data, - grpc_chttp2_stream_global *stream_global)) { - grpc_chttp2_stream *s; - grpc_chttp2_transport *t = TRANSPORT_FROM_GLOBAL(transport_global); - for (s = t->lists[GRPC_CHTTP2_LIST_ALL_STREAMS].head; s != NULL; - s = s->links[GRPC_CHTTP2_LIST_ALL_STREAMS].next) { - cb(transport_global, user_data, &s->global); - } +void grpc_chttp2_list_remove_stalled_by_transport(grpc_chttp2_transport *t, + grpc_chttp2_stream *s) { + stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); } diff --git a/src/core/ext/transport/chttp2/transport/stream_map.c b/src/core/ext/transport/chttp2/transport/stream_map.c index f70791c422..5f5a28446d 100644 --- a/src/core/ext/transport/chttp2/transport/stream_map.c +++ b/src/core/ext/transport/chttp2/transport/stream_map.c @@ -77,6 +77,7 @@ void grpc_chttp2_stream_map_add(grpc_chttp2_stream_map *map, uint32_t key, GPR_ASSERT(count == 0 || keys[count - 1] < key); GPR_ASSERT(value); + GPR_ASSERT(grpc_chttp2_stream_map_find(map, key) == NULL); if (count == capacity) { if (map->free > capacity / 4) { @@ -96,40 +97,6 @@ void grpc_chttp2_stream_map_add(grpc_chttp2_stream_map *map, uint32_t key, map->count = count + 1; } -void grpc_chttp2_stream_map_move_into(grpc_chttp2_stream_map *src, - grpc_chttp2_stream_map *dst) { - /* if src is empty we dont need to do anything */ - if (src->count == src->free) { - return; - } - /* if dst is empty we simply need to swap */ - if (dst->count == dst->free) { - GPR_SWAP(grpc_chttp2_stream_map, *src, *dst); - return; - } - /* the first element of src must be greater than the last of dst... - * however the maps may need compacting for this property to hold */ - if (src->keys[0] <= dst->keys[dst->count - 1]) { - src->count = compact(src->keys, src->values, src->count); - src->free = 0; - dst->count = compact(dst->keys, dst->values, dst->count); - dst->free = 0; - } - GPR_ASSERT(src->keys[0] > dst->keys[dst->count - 1]); - /* if dst doesn't have capacity, resize */ - if (dst->count + src->count > dst->capacity) { - dst->capacity = GPR_MAX(dst->capacity * 3 / 2, dst->count + src->count); - dst->keys = gpr_realloc(dst->keys, dst->capacity * sizeof(uint32_t)); - dst->values = gpr_realloc(dst->values, dst->capacity * sizeof(void *)); - } - memcpy(dst->keys + dst->count, src->keys, src->count * sizeof(uint32_t)); - memcpy(dst->values + dst->count, src->values, src->count * sizeof(void *)); - dst->count += src->count; - dst->free += src->free; - src->count = 0; - src->free = 0; -} - static void **find(grpc_chttp2_stream_map *map, uint32_t key) { size_t min_idx = 0; size_t max_idx = map->count; @@ -170,6 +137,7 @@ void *grpc_chttp2_stream_map_delete(grpc_chttp2_stream_map *map, uint32_t key) { if (map->free == map->count) { map->free = map->count = 0; } + GPR_ASSERT(grpc_chttp2_stream_map_find(map, key) == NULL); } return out; } @@ -183,6 +151,17 @@ size_t grpc_chttp2_stream_map_size(grpc_chttp2_stream_map *map) { return map->count - map->free; } +void *grpc_chttp2_stream_map_rand(grpc_chttp2_stream_map *map) { + if (map->count == map->free) { + return NULL; + } + if (map->free != 0) { + map->count = compact(map->keys, map->values, map->count); + map->free = 0; + } + return map->values[((size_t)rand()) % map->count]; +} + void grpc_chttp2_stream_map_for_each(grpc_chttp2_stream_map *map, void (*f)(void *user_data, uint32_t key, void *value), diff --git a/src/core/ext/transport/chttp2/transport/stream_map.h b/src/core/ext/transport/chttp2/transport/stream_map.h index b1d59ca6a3..203f640680 100644 --- a/src/core/ext/transport/chttp2/transport/stream_map.h +++ b/src/core/ext/transport/chttp2/transport/stream_map.h @@ -65,13 +65,12 @@ void grpc_chttp2_stream_map_add(grpc_chttp2_stream_map *map, uint32_t key, or NULL otherwise */ void *grpc_chttp2_stream_map_delete(grpc_chttp2_stream_map *map, uint32_t key); -/* Move all elements of src into dst */ -void grpc_chttp2_stream_map_move_into(grpc_chttp2_stream_map *src, - grpc_chttp2_stream_map *dst); - /* Return an existing key, or NULL if it does not exist */ void *grpc_chttp2_stream_map_find(grpc_chttp2_stream_map *map, uint32_t key); +/* Return a random entry */ +void *grpc_chttp2_stream_map_rand(grpc_chttp2_stream_map *map); + /* How many (populated) entries are in the stream map? */ size_t grpc_chttp2_stream_map_size(grpc_chttp2_stream_map *map); diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 979515bd54..139e7387c4 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -40,349 +40,221 @@ #include "src/core/ext/transport/chttp2/transport/http2_errors.h" #include "src/core/lib/profiling/timers.h" -static void finalize_outbuf(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_writing *transport_writing); +static void add_to_write_list(grpc_chttp2_write_cb **list, + grpc_chttp2_write_cb *cb) { + cb->next = *list; + *list = cb; +} + +static void finish_write_cb(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_stream *s, grpc_chttp2_write_cb *cb, + grpc_error *error) { + grpc_chttp2_complete_closure_step(exec_ctx, t, s, &cb->closure, error, + "finish_write_cb"); + cb->next = t->write_cb_pool; + t->write_cb_pool = cb; +} -int grpc_chttp2_unlocking_check_writes( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_writing *transport_writing) { - grpc_chttp2_stream_global *stream_global; - grpc_chttp2_stream_writing *stream_writing; +static void update_list(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_chttp2_stream *s, int64_t send_bytes, + grpc_chttp2_write_cb **list, grpc_error *error) { + grpc_chttp2_write_cb *cb = *list; + *list = NULL; + s->flow_controlled_bytes_written += send_bytes; + while (cb) { + grpc_chttp2_write_cb *next = cb->next; + if (cb->call_at_byte <= s->flow_controlled_bytes_written) { + finish_write_cb(exec_ctx, t, s, cb, GRPC_ERROR_REF(error)); + } else { + add_to_write_list(list, cb); + } + cb = next; + } + GRPC_ERROR_UNREF(error); +} - GPR_TIMER_BEGIN("grpc_chttp2_unlocking_check_writes", 0); +bool grpc_chttp2_begin_write(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { + grpc_chttp2_stream *s; - transport_writing->max_frame_size = - transport_global->settings[GRPC_ACKED_SETTINGS] - [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE]; + GPR_TIMER_BEGIN("grpc_chttp2_begin_write", 0); - if (transport_global->dirtied_local_settings && - !transport_global->sent_local_settings) { - gpr_slice_buffer_add( - &transport_writing->outbuf, + if (t->dirtied_local_settings && !t->sent_local_settings) { + grpc_slice_buffer_add( + &t->outbuf, grpc_chttp2_settings_create( - transport_global->settings[GRPC_SENT_SETTINGS], - transport_global->settings[GRPC_LOCAL_SETTINGS], - transport_global->force_send_settings, GRPC_CHTTP2_NUM_SETTINGS)); - transport_global->force_send_settings = 0; - transport_global->dirtied_local_settings = 0; - transport_global->sent_local_settings = 1; + t->settings[GRPC_SENT_SETTINGS], t->settings[GRPC_LOCAL_SETTINGS], + t->force_send_settings, GRPC_CHTTP2_NUM_SETTINGS)); + t->force_send_settings = 0; + t->dirtied_local_settings = 0; + t->sent_local_settings = 1; } /* simple writes are queued to qbuf, and flushed here */ - gpr_slice_buffer_move_into(&transport_global->qbuf, - &transport_writing->outbuf); - GPR_ASSERT(transport_global->qbuf.count == 0); + grpc_slice_buffer_move_into(&t->qbuf, &t->outbuf); + GPR_ASSERT(t->qbuf.count == 0); grpc_chttp2_hpack_compressor_set_max_table_size( - &transport_writing->hpack_compressor, - transport_global->settings[GRPC_PEER_SETTINGS] - [GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]); + &t->hpack_compressor, + t->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]); - GRPC_CHTTP2_FLOW_MOVE_TRANSPORT("write", transport_writing, outgoing_window, - transport_global, outgoing_window); - if (transport_writing->outgoing_window > 0) { - while (grpc_chttp2_list_pop_stalled_by_transport(transport_global, - &stream_global)) { - grpc_chttp2_become_writable(exec_ctx, transport_global, stream_global, - false, "transport.read_flow_control"); + if (t->outgoing_window > 0) { + while (grpc_chttp2_list_pop_stalled_by_transport(t, &s)) { + grpc_chttp2_become_writable(exec_ctx, t, s, false, + "transport.read_flow_control"); } } /* for each grpc_chttp2_stream that's become writable, frame it's data (according to available window sizes) and add to the output buffer */ - while (grpc_chttp2_list_pop_writable_stream( - transport_global, transport_writing, &stream_global, &stream_writing)) { - bool sent_initial_metadata = stream_writing->sent_initial_metadata; - bool become_writable = false; + while (grpc_chttp2_list_pop_writable_stream(t, &s)) { + bool sent_initial_metadata = s->sent_initial_metadata; + bool now_writing = false; - stream_writing->id = stream_global->id; - stream_writing->read_closed = stream_global->read_closed; + GRPC_CHTTP2_IF_TRACING(gpr_log( + GPR_DEBUG, "W:%p %s[%d] im-(sent,send)=(%d,%d) announce=%d", t, + t->is_client ? "CLIENT" : "SERVER", s->id, sent_initial_metadata, + s->send_initial_metadata != NULL, s->announce_window)); - GRPC_CHTTP2_FLOW_MOVE_STREAM("write", transport_writing, stream_writing, - outgoing_window, stream_global, - outgoing_window); - - if (!sent_initial_metadata && stream_global->send_initial_metadata) { - stream_writing->send_initial_metadata = - stream_global->send_initial_metadata; - stream_global->send_initial_metadata = NULL; - become_writable = true; + /* send initial metadata if it's available */ + if (!sent_initial_metadata && s->send_initial_metadata) { + grpc_chttp2_encode_header( + &t->hpack_compressor, s->id, s->send_initial_metadata, 0, + t->settings[GRPC_ACKED_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE], + &s->stats.outgoing, &t->outbuf); + s->send_initial_metadata = NULL; + s->sent_initial_metadata = true; sent_initial_metadata = true; + now_writing = true; + } + /* send any window updates */ + if (s->announce_window > 0) { + uint32_t announce = s->announce_window; + grpc_slice_buffer_add(&t->outbuf, + grpc_chttp2_window_update_create( + s->id, s->announce_window, &s->stats.outgoing)); + GRPC_CHTTP2_FLOW_DEBIT_STREAM("write", t, s, announce_window, announce); } if (sent_initial_metadata) { - if (stream_global->send_message != NULL) { - gpr_slice hdr = gpr_slice_malloc(5); - uint8_t *p = GPR_SLICE_START_PTR(hdr); - uint32_t len = stream_global->send_message->length; - GPR_ASSERT(stream_writing->send_message == NULL); - p[0] = (stream_global->send_message->flags & - GRPC_WRITE_INTERNAL_COMPRESS) != 0; - p[1] = (uint8_t)(len >> 24); - p[2] = (uint8_t)(len >> 16); - p[3] = (uint8_t)(len >> 8); - p[4] = (uint8_t)(len); - gpr_slice_buffer_add(&stream_writing->flow_controlled_buffer, hdr); - if (stream_global->send_message->length > 0) { - stream_writing->send_message = stream_global->send_message; - } else { - stream_writing->send_message = NULL; + /* send any body bytes, if allowed by flow control */ + if (s->flow_controlled_buffer.length > 0) { + uint32_t max_outgoing = + (uint32_t)GPR_MIN(t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE], + GPR_MIN(s->outgoing_window, t->outgoing_window)); + if (max_outgoing > 0) { + uint32_t send_bytes = + (uint32_t)GPR_MIN(max_outgoing, s->flow_controlled_buffer.length); + bool is_last_data_frame = + s->fetching_send_message == NULL && + send_bytes == s->flow_controlled_buffer.length; + bool is_last_frame = + is_last_data_frame && s->send_trailing_metadata != NULL && + grpc_metadata_batch_is_empty(s->send_trailing_metadata); + grpc_chttp2_encode_data(s->id, &s->flow_controlled_buffer, send_bytes, + is_last_frame, &s->stats.outgoing, + &t->outbuf); + GRPC_CHTTP2_FLOW_DEBIT_STREAM("write", t, s, outgoing_window, + send_bytes); + GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("write", t, outgoing_window, + send_bytes); + if (is_last_frame) { + s->send_trailing_metadata = NULL; + s->sent_trailing_metadata = true; + if (!t->is_client && !s->read_closed) { + grpc_slice_buffer_add(&t->outbuf, grpc_chttp2_rst_stream_create( + s->id, GRPC_CHTTP2_NO_ERROR, + &s->stats.outgoing)); + } + } + s->sending_bytes += send_bytes; + now_writing = true; + if (s->flow_controlled_buffer.length > 0) { + GRPC_CHTTP2_STREAM_REF(s, "chttp2_writing:fork"); + grpc_chttp2_list_add_writable_stream(t, s); + } + } else if (t->outgoing_window == 0) { + grpc_chttp2_list_add_stalled_by_transport(t, s); + now_writing = true; } - stream_writing->stream_fetched = 0; - stream_global->send_message = NULL; } - if ((stream_writing->send_message != NULL || - stream_writing->flow_controlled_buffer.length > 0) && - stream_writing->outgoing_window > 0) { - if (transport_writing->outgoing_window > 0) { - become_writable = true; + if (s->send_trailing_metadata != NULL && + s->fetching_send_message == NULL && + s->flow_controlled_buffer.length == 0) { + if (grpc_metadata_batch_is_empty(s->send_trailing_metadata)) { + grpc_chttp2_encode_data(s->id, &s->flow_controlled_buffer, 0, true, + &s->stats.outgoing, &t->outbuf); } else { - grpc_chttp2_list_add_stalled_by_transport(transport_writing, - stream_writing); + grpc_chttp2_encode_header( + &t->hpack_compressor, s->id, s->send_trailing_metadata, true, + t->settings[GRPC_ACKED_SETTINGS] + [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE], + &s->stats.outgoing, &t->outbuf); } + s->send_trailing_metadata = NULL; + s->sent_trailing_metadata = true; + if (!t->is_client && !s->read_closed) { + grpc_slice_buffer_add( + &t->outbuf, grpc_chttp2_rst_stream_create( + s->id, GRPC_CHTTP2_NO_ERROR, &s->stats.outgoing)); + } + now_writing = true; } - if (stream_global->send_trailing_metadata) { - stream_writing->send_trailing_metadata = - stream_global->send_trailing_metadata; - stream_global->send_trailing_metadata = NULL; - become_writable = true; - } - } - - if (!stream_global->read_closed && - stream_global->unannounced_incoming_window_for_writing > 1024) { - GRPC_CHTTP2_FLOW_MOVE_STREAM("write", transport_global, stream_writing, - announce_window, stream_global, - unannounced_incoming_window_for_writing); - become_writable = true; } - if (become_writable) { - grpc_chttp2_list_add_writing_stream(transport_writing, stream_writing); + if (now_writing) { + if (!grpc_chttp2_list_add_writing_stream(t, s)) { + /* already in writing list: drop ref */ + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:already_writing"); + } } else { - GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2_writing"); + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:no_write"); } } /* if the grpc_chttp2_transport is ready to send a window update, do so here also; 3/4 is a magic number that will likely get tuned soon */ - if (transport_global->announce_incoming_window > 0) { - uint32_t announced = (uint32_t)GPR_MIN( - transport_global->announce_incoming_window, UINT32_MAX); - GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("write", transport_global, - announce_incoming_window, announced); + if (t->announce_incoming_window > 0) { + uint32_t announced = + (uint32_t)GPR_MIN(t->announce_incoming_window, UINT32_MAX); + GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("write", t, announce_incoming_window, + announced); grpc_transport_one_way_stats throwaway_stats; - gpr_slice_buffer_add( - &transport_writing->outbuf, - grpc_chttp2_window_update_create(0, announced, &throwaway_stats)); + grpc_slice_buffer_add(&t->outbuf, grpc_chttp2_window_update_create( + 0, announced, &throwaway_stats)); } - GPR_TIMER_END("grpc_chttp2_unlocking_check_writes", 0); + GPR_TIMER_END("grpc_chttp2_begin_write", 0); - return transport_writing->outbuf.count > 0 || - grpc_chttp2_list_have_writing_streams(transport_writing); + return t->outbuf.count > 0; } -void grpc_chttp2_perform_writes( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_writing *transport_writing, - grpc_endpoint *endpoint) { - GPR_ASSERT(transport_writing->outbuf.count > 0 || - grpc_chttp2_list_have_writing_streams(transport_writing)); +void grpc_chttp2_end_write(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, + grpc_error *error) { + GPR_TIMER_BEGIN("grpc_chttp2_end_write", 0); + grpc_chttp2_stream *s; - finalize_outbuf(exec_ctx, transport_writing); - - GPR_ASSERT(endpoint); - - if (transport_writing->outbuf.count > 0) { - grpc_endpoint_write(exec_ctx, endpoint, &transport_writing->outbuf, - &transport_writing->done_cb); - } else { - grpc_exec_ctx_sched(exec_ctx, &transport_writing->done_cb, GRPC_ERROR_NONE, - NULL); - } -} - -static void finalize_outbuf(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport_writing *transport_writing) { - grpc_chttp2_stream_writing *stream_writing; - - GPR_TIMER_BEGIN("finalize_outbuf", 0); - - bool is_first_data_frame = true; - while ( - grpc_chttp2_list_pop_writing_stream(transport_writing, &stream_writing)) { - uint32_t max_outgoing = - (uint32_t)GPR_MIN(transport_writing->max_frame_size, - GPR_MIN(stream_writing->outgoing_window, - transport_writing->outgoing_window)); - /* send initial metadata if it's available */ - if (stream_writing->send_initial_metadata != NULL) { - grpc_chttp2_encode_header( - &transport_writing->hpack_compressor, stream_writing->id, - stream_writing->send_initial_metadata, 0, - transport_writing->max_frame_size, &stream_writing->stats, - &transport_writing->outbuf); - stream_writing->send_initial_metadata = NULL; - stream_writing->sent_initial_metadata = 1; - } - /* send any window updates */ - if (stream_writing->announce_window > 0 && - stream_writing->send_initial_metadata == NULL) { - uint32_t announce = stream_writing->announce_window; - gpr_slice_buffer_add( - &transport_writing->outbuf, - grpc_chttp2_window_update_create(stream_writing->id, - stream_writing->announce_window, - &stream_writing->stats)); - GRPC_CHTTP2_FLOW_DEBIT_STREAM("write", transport_writing, stream_writing, - announce_window, announce); - stream_writing->announce_window = 0; - } - /* fetch any body bytes */ - while (!stream_writing->fetching && stream_writing->send_message && - stream_writing->flow_controlled_buffer.length < max_outgoing && - stream_writing->stream_fetched < - stream_writing->send_message->length) { - if (grpc_byte_stream_next(exec_ctx, stream_writing->send_message, - &stream_writing->fetching_slice, max_outgoing, - &stream_writing->finished_fetch)) { - stream_writing->stream_fetched += - GPR_SLICE_LENGTH(stream_writing->fetching_slice); - if (stream_writing->stream_fetched == - stream_writing->send_message->length) { - stream_writing->send_message = NULL; - } - gpr_slice_buffer_add(&stream_writing->flow_controlled_buffer, - stream_writing->fetching_slice); - } else { - stream_writing->fetching = 1; - } - } - /* send any body bytes */ - if (stream_writing->flow_controlled_buffer.length > 0) { - if (max_outgoing > 0) { - uint32_t send_bytes = (uint32_t)GPR_MIN( - max_outgoing, stream_writing->flow_controlled_buffer.length); - int is_last_data_frame = - stream_writing->send_message == NULL && - send_bytes == stream_writing->flow_controlled_buffer.length; - int is_last_frame = is_last_data_frame && - stream_writing->send_trailing_metadata != NULL && - grpc_metadata_batch_is_empty( - stream_writing->send_trailing_metadata); - grpc_chttp2_encode_data( - stream_writing->id, &stream_writing->flow_controlled_buffer, - send_bytes, is_last_frame, &stream_writing->stats, - &transport_writing->outbuf); - if (is_first_data_frame) { - /* TODO(dgq): this is a hack. It'll be fix in a future refactoring */ - stream_writing->stats.data_bytes -= 5; /* discount grpc framing */ - is_first_data_frame = false; - } - GRPC_CHTTP2_FLOW_DEBIT_STREAM("write", transport_writing, - stream_writing, outgoing_window, - send_bytes); - GRPC_CHTTP2_FLOW_DEBIT_TRANSPORT("write", transport_writing, - outgoing_window, send_bytes); - if (is_last_frame) { - stream_writing->send_trailing_metadata = NULL; - stream_writing->sent_trailing_metadata = 1; - } - if (is_last_data_frame) { - GPR_ASSERT(stream_writing->send_message == NULL); - stream_writing->sent_message = 1; - } - } else if (transport_writing->outgoing_window == 0) { - grpc_chttp2_list_add_writing_stalled_by_transport(transport_writing, - stream_writing); - grpc_chttp2_list_add_written_stream(transport_writing, stream_writing); - } - } - /* send trailing metadata if it's available and we're ready for it */ - if (stream_writing->send_message == NULL && - stream_writing->flow_controlled_buffer.length == 0 && - stream_writing->send_trailing_metadata != NULL) { - if (grpc_metadata_batch_is_empty( - stream_writing->send_trailing_metadata)) { - grpc_chttp2_encode_data( - stream_writing->id, &stream_writing->flow_controlled_buffer, 0, 1, - &stream_writing->stats, &transport_writing->outbuf); - } else { - grpc_chttp2_encode_header( - &transport_writing->hpack_compressor, stream_writing->id, - stream_writing->send_trailing_metadata, 1, - transport_writing->max_frame_size, &stream_writing->stats, - &transport_writing->outbuf); - } - if (!transport_writing->is_client && !stream_writing->read_closed) { - gpr_slice_buffer_add(&transport_writing->outbuf, - grpc_chttp2_rst_stream_create( - stream_writing->id, GRPC_CHTTP2_NO_ERROR, - &stream_writing->stats)); - } - stream_writing->send_trailing_metadata = NULL; - stream_writing->sent_trailing_metadata = 1; - } - /* if there's more to write, then loop, otherwise prepare to finish the - * write */ - if ((stream_writing->flow_controlled_buffer.length > 0 || - (stream_writing->send_message && !stream_writing->fetching)) && - stream_writing->outgoing_window > 0) { - if (transport_writing->outgoing_window > 0) { - grpc_chttp2_list_add_writing_stream(transport_writing, stream_writing); - } else { - grpc_chttp2_list_add_writing_stalled_by_transport(transport_writing, - stream_writing); - grpc_chttp2_list_add_written_stream(transport_writing, stream_writing); - } - } else { - grpc_chttp2_list_add_written_stream(transport_writing, stream_writing); - } - } - - GPR_TIMER_END("finalize_outbuf", 0); -} - -void grpc_chttp2_cleanup_writing( - grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global, - grpc_chttp2_transport_writing *transport_writing) { - GPR_TIMER_BEGIN("grpc_chttp2_cleanup_writing", 0); - grpc_chttp2_stream_writing *stream_writing; - grpc_chttp2_stream_global *stream_global; - - if (grpc_chttp2_list_flush_writing_stalled_by_transport(exec_ctx, - transport_writing)) { - grpc_chttp2_initiate_write(exec_ctx, transport_global, false, - "resume_stalled_stream"); - } - - while (grpc_chttp2_list_pop_written_stream( - transport_global, transport_writing, &stream_global, &stream_writing)) { - if (stream_writing->sent_initial_metadata) { + while (grpc_chttp2_list_pop_writing_stream(t, &s)) { + if (s->sent_initial_metadata) { grpc_chttp2_complete_closure_step( - exec_ctx, transport_global, stream_global, - &stream_global->send_initial_metadata_finished, GRPC_ERROR_NONE); + exec_ctx, t, s, &s->send_initial_metadata_finished, + GRPC_ERROR_REF(error), "send_initial_metadata_finished"); } - grpc_transport_move_one_way_stats(&stream_writing->stats, - &stream_global->stats.outgoing); - if (stream_writing->sent_message) { - GPR_ASSERT(stream_writing->send_message == NULL); - grpc_chttp2_complete_closure_step( - exec_ctx, transport_global, stream_global, - &stream_global->send_message_finished, GRPC_ERROR_NONE); - stream_writing->sent_message = 0; + if (s->sending_bytes != 0) { + update_list(exec_ctx, t, s, (int64_t)s->sending_bytes, + &s->on_write_finished_cbs, GRPC_ERROR_REF(error)); + s->sending_bytes = 0; } - if (stream_writing->sent_trailing_metadata) { + if (s->sent_trailing_metadata) { grpc_chttp2_complete_closure_step( - exec_ctx, transport_global, stream_global, - &stream_global->send_trailing_metadata_finished, GRPC_ERROR_NONE); - } - if (stream_writing->sent_trailing_metadata) { - grpc_chttp2_mark_stream_closed(exec_ctx, transport_global, stream_global, - !transport_global->is_client, 1, - GRPC_ERROR_NONE); + exec_ctx, t, s, &s->send_trailing_metadata_finished, + GRPC_ERROR_REF(error), "send_trailing_metadata_finished"); + grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1, + GRPC_ERROR_REF(error)); } - GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2_writing"); + GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:end"); } - gpr_slice_buffer_reset_and_unref(&transport_writing->outbuf); - GPR_TIMER_END("grpc_chttp2_cleanup_writing", 0); + grpc_slice_buffer_reset_and_unref(&t->outbuf); + GRPC_ERROR_UNREF(error); + GPR_TIMER_END("grpc_chttp2_end_write", 0); } |